2024-12-04 04:28:14,373 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-04 04:28:14,394 main DEBUG Took 0.018480 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 04:28:14,394 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 04:28:14,395 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 04:28:14,396 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 04:28:14,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,416 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 04:28:14,433 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,434 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,435 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,436 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,437 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,438 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,438 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,439 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,439 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,441 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,442 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 04:28:14,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,443 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 04:28:14,445 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 04:28:14,447 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 04:28:14,448 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 04:28:14,449 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 04:28:14,451 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 04:28:14,451 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 04:28:14,461 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 04:28:14,464 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 04:28:14,466 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 04:28:14,466 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 04:28:14,467 main DEBUG createAppenders(={Console}) 2024-12-04 04:28:14,468 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 initialized 2024-12-04 04:28:14,468 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-04 04:28:14,469 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 OK. 2024-12-04 04:28:14,469 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 04:28:14,470 main DEBUG OutputStream closed 2024-12-04 04:28:14,470 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 04:28:14,470 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 04:28:14,471 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54a7079e OK 2024-12-04 04:28:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 04:28:14,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 04:28:14,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 04:28:14,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 04:28:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 04:28:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 04:28:14,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 04:28:14,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 04:28:14,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 04:28:14,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 04:28:14,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 04:28:14,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 04:28:14,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 04:28:14,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 04:28:14,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 04:28:14,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 04:28:14,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 04:28:14,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 04:28:14,577 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 04:28:14,577 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19b93fa8) with optional ClassLoader: null 2024-12-04 04:28:14,578 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 04:28:14,579 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19b93fa8] started OK. 2024-12-04T04:28:14,870 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a 2024-12-04 04:28:14,874 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 04:28:14,874 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T04:28:14,885 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-04T04:28:14,893 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-04T04:28:14,925 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T04:28:15,000 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T04:28:15,001 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T04:28:15,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T04:28:15,040 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41, deleteOnExit=true 2024-12-04T04:28:15,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T04:28:15,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/test.cache.data in system properties and HBase conf 2024-12-04T04:28:15,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T04:28:15,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir in system properties and HBase conf 2024-12-04T04:28:15,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T04:28:15,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T04:28:15,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T04:28:15,140 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T04:28:15,276 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T04:28:15,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T04:28:15,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T04:28:15,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T04:28:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T04:28:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T04:28:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T04:28:15,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T04:28:15,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T04:28:15,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T04:28:15,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/nfs.dump.dir in system properties and HBase conf 2024-12-04T04:28:15,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/java.io.tmpdir in system properties and HBase conf 2024-12-04T04:28:15,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T04:28:15,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T04:28:15,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T04:28:16,894 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T04:28:16,957 INFO [Time-limited test {}] log.Log(170): Logging initialized @3708ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T04:28:17,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T04:28:17,083 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T04:28:17,112 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T04:28:17,112 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T04:28:17,114 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T04:28:17,135 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T04:28:17,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,AVAILABLE} 2024-12-04T04:28:17,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T04:28:17,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/java.io.tmpdir/jetty-localhost-35277-hadoop-hdfs-3_4_1-tests_jar-_-any-6900625596294719980/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T04:28:17,379 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:35277} 2024-12-04T04:28:17,380 INFO [Time-limited test {}] server.Server(415): Started @4131ms 2024-12-04T04:28:18,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T04:28:18,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T04:28:18,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T04:28:18,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T04:28:18,144 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T04:28:18,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f76f489{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,AVAILABLE} 2024-12-04T04:28:18,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@433df981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T04:28:18,271 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@36632d60{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/java.io.tmpdir/jetty-localhost-39785-hadoop-hdfs-3_4_1-tests_jar-_-any-18194312148334752507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:28:18,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@751d2fa4{HTTP/1.1, (http/1.1)}{localhost:39785} 2024-12-04T04:28:18,272 INFO [Time-limited test {}] server.Server(415): Started @5024ms 2024-12-04T04:28:18,341 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T04:28:18,574 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T04:28:18,595 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T04:28:18,617 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T04:28:18,617 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T04:28:18,617 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T04:28:18,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b5fc47c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,AVAILABLE} 2024-12-04T04:28:18,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ebbf344{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T04:28:18,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4546bb60{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/java.io.tmpdir/jetty-localhost-33373-hadoop-hdfs-3_4_1-tests_jar-_-any-2760724264965809442/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:28:18,768 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c7d32f8{HTTP/1.1, (http/1.1)}{localhost:33373} 2024-12-04T04:28:18,768 INFO [Time-limited test {}] server.Server(415): Started @5520ms 2024-12-04T04:28:18,772 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T04:28:18,850 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T04:28:18,860 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T04:28:18,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T04:28:18,864 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T04:28:18,865 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T04:28:18,869 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da22a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,AVAILABLE} 2024-12-04T04:28:18,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f079a76{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T04:28:19,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43206bef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/java.io.tmpdir/jetty-localhost-44075-hadoop-hdfs-3_4_1-tests_jar-_-any-17772773132614303790/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:28:19,003 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@228ffa29{HTTP/1.1, (http/1.1)}{localhost:44075} 2024-12-04T04:28:19,004 INFO [Time-limited test {}] server.Server(415): Started @5755ms 2024-12-04T04:28:19,006 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T04:28:20,432 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data3/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,435 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data4/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,449 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data1/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,456 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data2/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,541 WARN [Thread-134 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data5/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,544 WARN [Thread-135 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data6/current/BP-90967970-172.17.0.3-1733286495978/current, will proceed with Du for space computation calculation, 2024-12-04T04:28:20,557 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T04:28:20,596 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T04:28:20,633 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T04:28:20,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc071b3dc33fe5e6d with lease ID 0x86023fe79767bdea: Processing first storage report for DS-9d637861-4190-4908-8983-0774d7a0d00a from datanode DatanodeRegistration(127.0.0.1:40387, datanodeUuid=16fde2f2-0eb2-4c6b-a3e1-474017a51da7, infoPort=41361, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc071b3dc33fe5e6d with lease ID 0x86023fe79767bdea: from storage DS-9d637861-4190-4908-8983-0774d7a0d00a node DatanodeRegistration(127.0.0.1:40387, datanodeUuid=16fde2f2-0eb2-4c6b-a3e1-474017a51da7, infoPort=41361, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f41381a9e324ed2 with lease ID 0x86023fe79767bdeb: Processing first storage report for DS-603f0531-f499-4df5-bf23-553f02acec2f from datanode DatanodeRegistration(127.0.0.1:38591, datanodeUuid=426f4c1d-e284-418e-b4c4-90e5c05285b1, infoPort=42201, infoSecurePort=0, ipcPort=43743, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f41381a9e324ed2 with lease ID 0x86023fe79767bdeb: from storage DS-603f0531-f499-4df5-bf23-553f02acec2f node DatanodeRegistration(127.0.0.1:38591, datanodeUuid=426f4c1d-e284-418e-b4c4-90e5c05285b1, infoPort=42201, infoSecurePort=0, ipcPort=43743, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x235ee610e9f28cc with lease ID 0x86023fe79767bdec: Processing first storage report for DS-31ae20d1-2601-4459-b770-2152aaba8d02 from datanode DatanodeRegistration(127.0.0.1:43031, datanodeUuid=9dc7fb19-548d-43b9-8e6a-9d510388cda0, infoPort=35321, infoSecurePort=0, ipcPort=45105, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x235ee610e9f28cc with lease ID 0x86023fe79767bdec: from storage DS-31ae20d1-2601-4459-b770-2152aaba8d02 node DatanodeRegistration(127.0.0.1:43031, datanodeUuid=9dc7fb19-548d-43b9-8e6a-9d510388cda0, infoPort=35321, infoSecurePort=0, ipcPort=45105, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f41381a9e324ed2 with lease ID 0x86023fe79767bdeb: Processing first storage report for DS-8f620ed3-b024-4365-8798-a995fc663b37 from datanode DatanodeRegistration(127.0.0.1:38591, datanodeUuid=426f4c1d-e284-418e-b4c4-90e5c05285b1, infoPort=42201, infoSecurePort=0, ipcPort=43743, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f41381a9e324ed2 with lease ID 0x86023fe79767bdeb: from storage DS-8f620ed3-b024-4365-8798-a995fc663b37 node DatanodeRegistration(127.0.0.1:38591, datanodeUuid=426f4c1d-e284-418e-b4c4-90e5c05285b1, infoPort=42201, infoSecurePort=0, ipcPort=43743, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc071b3dc33fe5e6d with lease ID 0x86023fe79767bdea: Processing first storage report for DS-8373eb03-9c8b-423a-826b-c59039e21758 from datanode DatanodeRegistration(127.0.0.1:40387, datanodeUuid=16fde2f2-0eb2-4c6b-a3e1-474017a51da7, infoPort=41361, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc071b3dc33fe5e6d with lease ID 0x86023fe79767bdea: from storage DS-8373eb03-9c8b-423a-826b-c59039e21758 node DatanodeRegistration(127.0.0.1:40387, datanodeUuid=16fde2f2-0eb2-4c6b-a3e1-474017a51da7, infoPort=41361, infoSecurePort=0, ipcPort=46445, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x235ee610e9f28cc with lease ID 0x86023fe79767bdec: Processing first storage report for DS-b2691e64-75f1-4cd1-a224-bffa3a932fd8 from datanode DatanodeRegistration(127.0.0.1:43031, datanodeUuid=9dc7fb19-548d-43b9-8e6a-9d510388cda0, infoPort=35321, infoSecurePort=0, ipcPort=45105, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978) 2024-12-04T04:28:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x235ee610e9f28cc with lease ID 0x86023fe79767bdec: from storage DS-b2691e64-75f1-4cd1-a224-bffa3a932fd8 node DatanodeRegistration(127.0.0.1:43031, datanodeUuid=9dc7fb19-548d-43b9-8e6a-9d510388cda0, infoPort=35321, infoSecurePort=0, ipcPort=45105, storageInfo=lv=-57;cid=testClusterID;nsid=2078769581;c=1733286495978), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T04:28:20,676 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a 2024-12-04T04:28:20,793 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/zookeeper_0, clientPort=50801, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T04:28:20,815 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50801 2024-12-04T04:28:20,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:20,835 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:21,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741825_1001 (size=7) 2024-12-04T04:28:21,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741825_1001 (size=7) 2024-12-04T04:28:21,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741825_1001 (size=7) 2024-12-04T04:28:21,610 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 with version=8 2024-12-04T04:28:21,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/hbase-staging 2024-12-04T04:28:22,020 INFO [Time-limited test {}] client.ConnectionUtils(128): master/67abd3a0ee76:0 server-side Connection retries=45 2024-12-04T04:28:22,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:22,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:22,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T04:28:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T04:28:22,269 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T04:28:22,369 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T04:28:22,383 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T04:28:22,388 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T04:28:22,429 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 66731 (auto-detected) 2024-12-04T04:28:22,430 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-04T04:28:22,472 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33125 2024-12-04T04:28:22,509 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33125 connecting to ZooKeeper ensemble=127.0.0.1:50801 2024-12-04T04:28:22,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331250x0, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T04:28:22,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33125-0x1019e9915430000 connected 2024-12-04T04:28:22,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:22,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:22,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:28:22,915 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205, hbase.cluster.distributed=false 2024-12-04T04:28:22,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T04:28:22,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33125 2024-12-04T04:28:22,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33125 2024-12-04T04:28:22,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33125 2024-12-04T04:28:23,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33125 2024-12-04T04:28:23,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33125 2024-12-04T04:28:23,193 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/67abd3a0ee76:0 server-side Connection retries=45 2024-12-04T04:28:23,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,197 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T04:28:23,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T04:28:23,201 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T04:28:23,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T04:28:23,211 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40991 2024-12-04T04:28:23,213 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40991 connecting to ZooKeeper ensemble=127.0.0.1:50801 2024-12-04T04:28:23,215 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409910x0, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T04:28:23,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40991-0x1019e9915430001 connected 2024-12-04T04:28:23,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:28:23,266 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T04:28:23,279 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T04:28:23,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T04:28:23,297 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T04:28:23,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40991 2024-12-04T04:28:23,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40991 2024-12-04T04:28:23,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40991 2024-12-04T04:28:23,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40991 2024-12-04T04:28:23,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40991 2024-12-04T04:28:23,327 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/67abd3a0ee76:0 server-side Connection retries=45 2024-12-04T04:28:23,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,328 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T04:28:23,328 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T04:28:23,329 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T04:28:23,330 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T04:28:23,331 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41303 2024-12-04T04:28:23,334 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41303 connecting to ZooKeeper ensemble=127.0.0.1:50801 2024-12-04T04:28:23,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413030x0, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T04:28:23,392 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41303-0x1019e9915430002 connected 2024-12-04T04:28:23,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:28:23,397 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T04:28:23,405 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T04:28:23,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T04:28:23,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T04:28:23,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41303 2024-12-04T04:28:23,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41303 2024-12-04T04:28:23,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41303 2024-12-04T04:28:23,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41303 2024-12-04T04:28:23,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41303 2024-12-04T04:28:23,448 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/67abd3a0ee76:0 server-side Connection retries=45 2024-12-04T04:28:23,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,449 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T04:28:23,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T04:28:23,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T04:28:23,450 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T04:28:23,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T04:28:23,452 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35953 2024-12-04T04:28:23,454 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35953 connecting to ZooKeeper ensemble=127.0.0.1:50801 2024-12-04T04:28:23,456 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,460 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359530x0, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T04:28:23,485 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35953-0x1019e9915430003 connected 2024-12-04T04:28:23,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:28:23,487 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T04:28:23,490 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T04:28:23,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T04:28:23,494 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T04:28:23,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35953 2024-12-04T04:28:23,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35953 2024-12-04T04:28:23,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35953 2024-12-04T04:28:23,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35953 2024-12-04T04:28:23,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35953 2024-12-04T04:28:23,543 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;67abd3a0ee76:33125 2024-12-04T04:28:23,544 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:23,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,569 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:23,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T04:28:23,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T04:28:23,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T04:28:23,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,613 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T04:28:23,614 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/67abd3a0ee76,33125,1733286501754 from backup master directory 2024-12-04T04:28:23,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:23,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T04:28:23,631 WARN [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T04:28:23,631 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:23,634 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T04:28:23,636 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T04:28:23,719 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/hbase.id] with ID: 436670bb-39cf-467e-9f7a-82d611d5e88b 2024-12-04T04:28:23,719 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/.tmp/hbase.id 2024-12-04T04:28:23,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741826_1002 (size=42) 2024-12-04T04:28:23,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741826_1002 (size=42) 2024-12-04T04:28:23,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741826_1002 (size=42) 2024-12-04T04:28:23,742 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/.tmp/hbase.id]:[hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/hbase.id] 2024-12-04T04:28:23,802 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T04:28:23,808 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T04:28:23,829 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-12-04T04:28:23,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:23,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741827_1003 (size=196) 2024-12-04T04:28:23,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741827_1003 (size=196) 2024-12-04T04:28:23,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741827_1003 (size=196) 2024-12-04T04:28:24,306 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T04:28:24,308 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T04:28:24,316 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741828_1004 (size=1189) 2024-12-04T04:28:24,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741828_1004 (size=1189) 2024-12-04T04:28:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741828_1004 (size=1189) 2024-12-04T04:28:24,802 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store 2024-12-04T04:28:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741829_1005 (size=34) 2024-12-04T04:28:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741829_1005 (size=34) 2024-12-04T04:28:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741829_1005 (size=34) 2024-12-04T04:28:24,838 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T04:28:24,842 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:24,844 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T04:28:24,845 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:28:24,845 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:28:24,847 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T04:28:24,847 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:28:24,847 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:28:24,849 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733286504844Disabling compacts and flushes for region at 1733286504844Disabling writes for close at 1733286504847 (+3 ms)Writing region close event to WAL at 1733286504847Closed at 1733286504847 2024-12-04T04:28:24,852 WARN [master/67abd3a0ee76:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/.initializing 2024-12-04T04:28:24,852 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/WALs/67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:24,861 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:24,881 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=67abd3a0ee76%2C33125%2C1733286501754, suffix=, logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/WALs/67abd3a0ee76,33125,1733286501754, archiveDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/oldWALs, maxLogs=10 2024-12-04T04:28:24,926 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/WALs/67abd3a0ee76,33125,1733286501754/67abd3a0ee76%2C33125%2C1733286501754.1733286504888, exclude list is [], retry=0 2024-12-04T04:28:24,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:24,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:24,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:24,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:24,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T04:28:25,016 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/WALs/67abd3a0ee76,33125,1733286501754/67abd3a0ee76%2C33125%2C1733286501754.1733286504888 2024-12-04T04:28:25,017 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:25,018 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:25,019 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:25,023 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,024 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T04:28:25,104 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:25,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T04:28:25,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:25,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T04:28:25,116 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,117 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:25,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T04:28:25,121 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:25,123 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,126 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,128 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,133 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,134 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,138 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T04:28:25,142 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T04:28:25,146 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:25,148 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68351390, jitterRate=0.018515080213546753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T04:28:25,153 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733286505041Initializing all the Stores at 1733286505043 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286505044 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286505045 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286505045Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286505045Cleaning up temporary data from old regions at 1733286505134 (+89 ms)Region opened successfully at 1733286505153 (+19 ms) 2024-12-04T04:28:25,154 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T04:28:25,190 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1b9552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=67abd3a0ee76/172.17.0.3:0 2024-12-04T04:28:25,221 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T04:28:25,232 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T04:28:25,233 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T04:28:25,235 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T04:28:25,237 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T04:28:25,242 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T04:28:25,242 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T04:28:25,268 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T04:28:25,278 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T04:28:25,292 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T04:28:25,295 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T04:28:25,297 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T04:28:25,302 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T04:28:25,305 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T04:28:25,309 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T04:28:25,321 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T04:28:25,323 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T04:28:25,355 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T04:28:25,377 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T04:28:25,397 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,433 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=67abd3a0ee76,33125,1733286501754, sessionid=0x1019e9915430000, setting cluster-up flag (Was=false) 2024-12-04T04:28:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,492 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T04:28:25,495 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:25,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:25,555 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T04:28:25,557 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:25,563 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T04:28:25,629 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(746): ClusterId : 436670bb-39cf-467e-9f7a-82d611d5e88b 2024-12-04T04:28:25,629 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(746): ClusterId : 436670bb-39cf-467e-9f7a-82d611d5e88b 2024-12-04T04:28:25,630 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(746): ClusterId : 436670bb-39cf-467e-9f7a-82d611d5e88b 2024-12-04T04:28:25,632 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T04:28:25,632 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T04:28:25,632 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T04:28:25,638 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T04:28:25,647 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T04:28:25,651 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T04:28:25,651 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T04:28:25,651 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T04:28:25,651 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T04:28:25,651 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T04:28:25,651 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T04:28:25,654 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T04:28:25,659 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 67abd3a0ee76,33125,1733286501754 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T04:28:25,672 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T04:28:25,672 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T04:28:25,672 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T04:28:25,672 DEBUG [RS:0;67abd3a0ee76:40991 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569b8ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=67abd3a0ee76/172.17.0.3:0 2024-12-04T04:28:25,672 DEBUG [RS:2;67abd3a0ee76:35953 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa83465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=67abd3a0ee76/172.17.0.3:0 2024-12-04T04:28:25,672 DEBUG [RS:1;67abd3a0ee76:41303 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b08dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=67abd3a0ee76/172.17.0.3:0 2024-12-04T04:28:25,674 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/67abd3a0ee76:0, corePoolSize=5, maxPoolSize=5 2024-12-04T04:28:25,674 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/67abd3a0ee76:0, corePoolSize=5, maxPoolSize=5 2024-12-04T04:28:25,674 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/67abd3a0ee76:0, corePoolSize=5, maxPoolSize=5 2024-12-04T04:28:25,674 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/67abd3a0ee76:0, corePoolSize=5, maxPoolSize=5 2024-12-04T04:28:25,674 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/67abd3a0ee76:0, corePoolSize=10, maxPoolSize=10 2024-12-04T04:28:25,675 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,675 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/67abd3a0ee76:0, corePoolSize=2, maxPoolSize=2 2024-12-04T04:28:25,675 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,686 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;67abd3a0ee76:35953 2024-12-04T04:28:25,689 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;67abd3a0ee76:41303 2024-12-04T04:28:25,690 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T04:28:25,690 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T04:28:25,691 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T04:28:25,691 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T04:28:25,691 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T04:28:25,691 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T04:28:25,691 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T04:28:25,691 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T04:28:25,692 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733286535692 2024-12-04T04:28:25,694 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;67abd3a0ee76:40991 2024-12-04T04:28:25,694 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T04:28:25,695 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T04:28:25,696 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(2659): reportForDuty to master=67abd3a0ee76,33125,1733286501754 with port=41303, startcode=1733286503326 2024-12-04T04:28:25,696 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(2659): reportForDuty to master=67abd3a0ee76,33125,1733286501754 with port=35953, startcode=1733286503448 2024-12-04T04:28:25,699 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T04:28:25,699 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T04:28:25,699 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T04:28:25,700 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T04:28:25,702 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,702 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T04:28:25,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T04:28:25,702 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T04:28:25,702 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T04:28:25,701 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,704 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(2659): reportForDuty to master=67abd3a0ee76,33125,1733286501754 with port=40991, startcode=1733286503128 2024-12-04T04:28:25,712 DEBUG [RS:2;67abd3a0ee76:35953 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T04:28:25,712 DEBUG [RS:0;67abd3a0ee76:40991 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T04:28:25,712 DEBUG [RS:1;67abd3a0ee76:41303 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T04:28:25,712 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T04:28:25,713 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T04:28:25,714 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T04:28:25,715 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T04:28:25,716 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T04:28:25,718 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.large.0-1733286505717,5,FailOnTimeoutGroup] 2024-12-04T04:28:25,724 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.small.0-1733286505718,5,FailOnTimeoutGroup] 2024-12-04T04:28:25,724 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,725 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T04:28:25,726 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,726 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741831_1007 (size=1321) 2024-12-04T04:28:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741831_1007 (size=1321) 2024-12-04T04:28:25,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741831_1007 (size=1321) 2024-12-04T04:28:25,781 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T04:28:25,782 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 2024-12-04T04:28:25,799 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45655, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T04:28:25,799 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33407, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T04:28:25,800 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T04:28:25,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741832_1008 (size=32) 2024-12-04T04:28:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741832_1008 (size=32) 2024-12-04T04:28:25,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741832_1008 (size=32) 2024-12-04T04:28:25,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:25,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:25,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T04:28:25,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(517): Registering regionserver=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:25,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T04:28:25,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:25,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T04:28:25,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T04:28:25,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:25,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T04:28:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(517): Registering regionserver=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:25,824 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T04:28:25,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,826 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 2024-12-04T04:28:25,827 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35021 2024-12-04T04:28:25,827 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T04:28:25,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:25,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T04:28:25,829 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:25,829 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33125 {}] master.ServerManager(517): Registering regionserver=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:25,830 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 2024-12-04T04:28:25,831 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35021 2024-12-04T04:28:25,831 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T04:28:25,831 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T04:28:25,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:25,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:25,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T04:28:25,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740 2024-12-04T04:28:25,835 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 2024-12-04T04:28:25,835 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35021 2024-12-04T04:28:25,835 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T04:28:25,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740 2024-12-04T04:28:25,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T04:28:25,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T04:28:25,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T04:28:25,840 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T04:28:25,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T04:28:25,847 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:25,848 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68427870, jitterRate=0.01965472102165222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T04:28:25,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733286505806Initializing all the Stores at 1733286505808 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286505808Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286505809 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286505809Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286505809Cleaning up temporary data from old regions at 1733286505839 (+30 ms)Region opened successfully at 1733286505850 (+11 ms) 2024-12-04T04:28:25,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T04:28:25,850 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T04:28:25,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T04:28:25,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T04:28:25,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T04:28:25,852 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T04:28:25,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733286505850Disabling compacts and flushes for region at 1733286505850Disabling writes for close at 1733286505850Writing region close event to WAL at 1733286505851 (+1 ms)Closed at 1733286505852 (+1 ms) 2024-12-04T04:28:25,854 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T04:28:25,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T04:28:25,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T04:28:25,870 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T04:28:25,874 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T04:28:25,916 DEBUG [RS:0;67abd3a0ee76:40991 {}] zookeeper.ZKUtil(111): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:25,916 WARN [RS:0;67abd3a0ee76:40991 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T04:28:25,916 DEBUG [RS:2;67abd3a0ee76:35953 {}] zookeeper.ZKUtil(111): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:25,917 WARN [RS:2;67abd3a0ee76:35953 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T04:28:25,917 INFO [RS:2;67abd3a0ee76:35953 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:25,917 DEBUG [RS:1;67abd3a0ee76:41303 {}] zookeeper.ZKUtil(111): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:25,917 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:25,917 WARN [RS:1;67abd3a0ee76:41303 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T04:28:25,917 INFO [RS:1;67abd3a0ee76:41303 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:25,916 INFO [RS:0;67abd3a0ee76:40991 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:25,917 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:25,917 DEBUG [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:25,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [67abd3a0ee76,35953,1733286503448] 2024-12-04T04:28:25,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [67abd3a0ee76,40991,1733286503128] 2024-12-04T04:28:25,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [67abd3a0ee76,41303,1733286503326] 2024-12-04T04:28:25,946 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T04:28:25,946 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T04:28:25,946 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T04:28:25,960 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T04:28:25,961 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T04:28:25,961 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T04:28:25,965 INFO [RS:2;67abd3a0ee76:35953 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T04:28:25,965 INFO [RS:0;67abd3a0ee76:40991 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T04:28:25,966 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,966 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,969 INFO [RS:1;67abd3a0ee76:41303 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T04:28:25,969 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,970 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T04:28:25,970 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T04:28:25,970 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T04:28:25,976 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T04:28:25,976 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T04:28:25,977 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T04:28:25,978 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,978 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0, corePoolSize=2, maxPoolSize=2 2024-12-04T04:28:25,979 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,979 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0, corePoolSize=2, maxPoolSize=2 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,980 DEBUG [RS:1;67abd3a0ee76:41303 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,980 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,981 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,981 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,981 DEBUG [RS:0;67abd3a0ee76:40991 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,980 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,982 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,982 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,982 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0, corePoolSize=2, maxPoolSize=2 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,983 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,984 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,984 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/67abd3a0ee76:0, corePoolSize=1, maxPoolSize=1 2024-12-04T04:28:25,984 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,984 DEBUG [RS:2;67abd3a0ee76:35953 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/67abd3a0ee76:0, corePoolSize=3, maxPoolSize=3 2024-12-04T04:28:25,991 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,991 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,991 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,991 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,992 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,992 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,41303,1733286503326-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,994 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,40991,1733286503128-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T04:28:25,996 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,996 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,997 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,997 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:25,999 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,000 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,35953,1733286503448-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T04:28:26,016 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T04:28:26,016 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T04:28:26,019 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T04:28:26,019 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,40991,1733286503128-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,019 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,41303,1733286503326-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,019 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,35953,1733286503448-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,019 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,019 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,019 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,020 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.Replication(171): 67abd3a0ee76,35953,1733286503448 started 2024-12-04T04:28:26,020 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.Replication(171): 67abd3a0ee76,41303,1733286503326 started 2024-12-04T04:28:26,020 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.Replication(171): 67abd3a0ee76,40991,1733286503128 started 2024-12-04T04:28:26,025 WARN [67abd3a0ee76:33125 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T04:28:26,045 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,045 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,045 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1482): Serving as 67abd3a0ee76,40991,1733286503128, RpcServer on 67abd3a0ee76/172.17.0.3:40991, sessionid=0x1019e9915430001 2024-12-04T04:28:26,045 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1482): Serving as 67abd3a0ee76,35953,1733286503448, RpcServer on 67abd3a0ee76/172.17.0.3:35953, sessionid=0x1019e9915430003 2024-12-04T04:28:26,046 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:26,046 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1482): Serving as 67abd3a0ee76,41303,1733286503326, RpcServer on 67abd3a0ee76/172.17.0.3:41303, sessionid=0x1019e9915430002 2024-12-04T04:28:26,046 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T04:28:26,046 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T04:28:26,046 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T04:28:26,047 DEBUG [RS:0;67abd3a0ee76:40991 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:26,047 DEBUG [RS:1;67abd3a0ee76:41303 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:26,047 DEBUG [RS:2;67abd3a0ee76:35953 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:26,047 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,41303,1733286503326' 2024-12-04T04:28:26,047 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,40991,1733286503128' 2024-12-04T04:28:26,047 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,35953,1733286503448' 2024-12-04T04:28:26,047 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T04:28:26,047 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T04:28:26,047 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T04:28:26,048 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T04:28:26,048 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T04:28:26,048 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T04:28:26,049 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T04:28:26,049 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T04:28:26,049 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T04:28:26,049 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T04:28:26,049 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T04:28:26,049 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T04:28:26,050 DEBUG [RS:0;67abd3a0ee76:40991 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:26,050 DEBUG [RS:1;67abd3a0ee76:41303 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:26,050 DEBUG [RS:2;67abd3a0ee76:35953 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:26,050 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,40991,1733286503128' 2024-12-04T04:28:26,050 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,35953,1733286503448' 2024-12-04T04:28:26,050 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '67abd3a0ee76,41303,1733286503326' 2024-12-04T04:28:26,050 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T04:28:26,050 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T04:28:26,050 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T04:28:26,052 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T04:28:26,052 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T04:28:26,052 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T04:28:26,052 DEBUG [RS:0;67abd3a0ee76:40991 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T04:28:26,053 INFO [RS:0;67abd3a0ee76:40991 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T04:28:26,053 DEBUG [RS:2;67abd3a0ee76:35953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T04:28:26,053 INFO [RS:0;67abd3a0ee76:40991 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T04:28:26,053 INFO [RS:2;67abd3a0ee76:35953 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T04:28:26,053 INFO [RS:2;67abd3a0ee76:35953 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T04:28:26,053 DEBUG [RS:1;67abd3a0ee76:41303 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T04:28:26,053 INFO [RS:1;67abd3a0ee76:41303 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T04:28:26,053 INFO [RS:1;67abd3a0ee76:41303 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T04:28:26,159 INFO [RS:1;67abd3a0ee76:41303 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:26,160 INFO [RS:0;67abd3a0ee76:40991 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:26,161 INFO [RS:2;67abd3a0ee76:35953 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:26,169 INFO [RS:0;67abd3a0ee76:40991 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=67abd3a0ee76%2C40991%2C1733286503128, suffix=, logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,40991,1733286503128, archiveDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs, maxLogs=32 2024-12-04T04:28:26,171 INFO [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=67abd3a0ee76%2C35953%2C1733286503448, suffix=, logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448, archiveDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs, maxLogs=32 2024-12-04T04:28:26,177 INFO [RS:1;67abd3a0ee76:41303 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=67abd3a0ee76%2C41303%2C1733286503326, suffix=, logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326, archiveDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs, maxLogs=32 2024-12-04T04:28:26,195 DEBUG [RS:2;67abd3a0ee76:35953 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448/67abd3a0ee76%2C35953%2C1733286503448.1733286506176, exclude list is [], retry=0 2024-12-04T04:28:26,198 DEBUG [RS:0;67abd3a0ee76:40991 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,40991,1733286503128/67abd3a0ee76%2C40991%2C1733286503128.1733286506176, exclude list is [], retry=0 2024-12-04T04:28:26,200 DEBUG [RS:1;67abd3a0ee76:41303 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, exclude list is [], retry=0 2024-12-04T04:28:26,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:26,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:26,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:26,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:26,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:26,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:26,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:26,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:26,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:26,260 INFO [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448/67abd3a0ee76%2C35953%2C1733286503448.1733286506176 2024-12-04T04:28:26,262 DEBUG [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:26,272 INFO [RS:0;67abd3a0ee76:40991 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,40991,1733286503128/67abd3a0ee76%2C40991%2C1733286503128.1733286506176 2024-12-04T04:28:26,273 INFO [RS:1;67abd3a0ee76:41303 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 2024-12-04T04:28:26,276 DEBUG [RS:0;67abd3a0ee76:40991 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:26,281 DEBUG [RS:1;67abd3a0ee76:41303 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:28:26,529 DEBUG [67abd3a0ee76:33125 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-04T04:28:26,537 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(204): Hosts are {67abd3a0ee76=0} racks are {/default-rack=0} 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T04:28:26,544 INFO [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T04:28:26,544 INFO [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T04:28:26,544 INFO [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T04:28:26,544 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T04:28:26,554 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:26,561 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 67abd3a0ee76,35953,1733286503448, state=OPENING 2024-12-04T04:28:26,618 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T04:28:26,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:26,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:26,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:26,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:28:26,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:26,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:26,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:26,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:26,635 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T04:28:26,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=67abd3a0ee76,35953,1733286503448}] 2024-12-04T04:28:26,817 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T04:28:26,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33553, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T04:28:26,831 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T04:28:26,831 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:26,832 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T04:28:26,835 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=67abd3a0ee76%2C35953%2C1733286503448.meta, suffix=.meta, logDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448, archiveDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs, maxLogs=32 2024-12-04T04:28:26,849 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448/67abd3a0ee76%2C35953%2C1733286503448.meta.1733286506837.meta, exclude list is [], retry=0 2024-12-04T04:28:26,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:26,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:26,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:26,856 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,35953,1733286503448/67abd3a0ee76%2C35953%2C1733286503448.meta.1733286506837.meta 2024-12-04T04:28:26,856 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:26,856 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:26,858 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T04:28:26,861 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T04:28:26,865 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T04:28:26,869 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T04:28:26,869 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:26,870 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T04:28:26,870 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T04:28:26,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T04:28:26,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T04:28:26,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:26,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:26,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T04:28:26,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T04:28:26,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:26,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:26,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T04:28:26,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T04:28:26,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:26,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:26,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T04:28:26,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T04:28:26,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:26,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T04:28:26,882 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T04:28:26,883 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740 2024-12-04T04:28:26,886 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740 2024-12-04T04:28:26,888 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T04:28:26,888 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T04:28:26,889 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T04:28:26,892 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T04:28:26,894 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72088269, jitterRate=0.07419891655445099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T04:28:26,894 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T04:28:26,895 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733286506870Writing region info on filesystem at 1733286506870Initializing all the Stores at 1733286506872 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286506872Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286506873 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286506873Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733286506873Cleaning up temporary data from old regions at 1733286506888 (+15 ms)Running coprocessor post-open hooks at 1733286506894 (+6 ms)Region opened successfully at 1733286506895 (+1 ms) 2024-12-04T04:28:26,901 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733286506809 2024-12-04T04:28:26,912 DEBUG [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T04:28:26,913 INFO [RS_OPEN_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T04:28:26,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:26,917 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 67abd3a0ee76,35953,1733286503448, state=OPEN 2024-12-04T04:28:27,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T04:28:27,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T04:28:27,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T04:28:27,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T04:28:27,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:27,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:27,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:27,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T04:28:27,117 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:27,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T04:28:27,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=67abd3a0ee76,35953,1733286503448 in 480 msec 2024-12-04T04:28:27,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T04:28:27,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2660 sec 2024-12-04T04:28:27,136 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T04:28:27,136 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T04:28:27,157 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T04:28:27,159 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=67abd3a0ee76,35953,1733286503448, seqNum=-1] 2024-12-04T04:28:27,207 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T04:28:27,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49337, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T04:28:27,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6490 sec 2024-12-04T04:28:27,250 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733286507250, completionTime=-1 2024-12-04T04:28:27,253 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-04T04:28:27,253 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T04:28:27,285 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-04T04:28:27,286 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733286567286 2024-12-04T04:28:27,286 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733286627286 2024-12-04T04:28:27,286 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-04T04:28:27,288 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T04:28:27,296 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,296 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,296 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,298 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-67abd3a0ee76:33125, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,299 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,299 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,306 DEBUG [master/67abd3a0ee76:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T04:28:27,336 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.705sec 2024-12-04T04:28:27,339 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T04:28:27,340 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T04:28:27,341 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T04:28:27,342 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T04:28:27,342 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T04:28:27,343 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T04:28:27,343 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T04:28:27,352 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T04:28:27,353 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T04:28:27,353 INFO [master/67abd3a0ee76:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=67abd3a0ee76,33125,1733286501754-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:27,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a40816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T04:28:27,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 67abd3a0ee76,33125,-1 for getting cluster id 2024-12-04T04:28:27,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T04:28:27,461 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '436670bb-39cf-467e-9f7a-82d611d5e88b' 2024-12-04T04:28:27,466 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T04:28:27,466 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "436670bb-39cf-467e-9f7a-82d611d5e88b" 2024-12-04T04:28:27,468 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17abd8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T04:28:27,468 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [67abd3a0ee76,33125,-1] 2024-12-04T04:28:27,473 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T04:28:27,478 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:28:27,481 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T04:28:27,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e243daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T04:28:27,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T04:28:27,506 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=67abd3a0ee76,35953,1733286503448, seqNum=-1] 2024-12-04T04:28:27,507 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T04:28:27,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T04:28:27,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:27,548 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35021/hbase 2024-12-04T04:28:27,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=360, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=792, ProcessCount=11, AvailableMemoryMB=6743 2024-12-04T04:28:27,594 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:27,597 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:27,598 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:27,604 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21678556, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-21678556, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:27,622 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21678556/hregion-21678556.1733286507605, exclude list is [], retry=0 2024-12-04T04:28:27,627 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:27,632 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:27,636 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:27,657 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21678556/hregion-21678556.1733286507605 2024-12-04T04:28:27,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:27,658 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 29997fa91edfab42405c6795d70f17b3, NAME => 'testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:27,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741838_1014 (size=64) 2024-12-04T04:28:27,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741838_1014 (size=64) 2024-12-04T04:28:27,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741838_1014 (size=64) 2024-12-04T04:28:27,693 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:27,696 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,700 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29997fa91edfab42405c6795d70f17b3 columnFamilyName a 2024-12-04T04:28:27,700 DEBUG [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:27,701 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] regionserver.HStore(327): Store=29997fa91edfab42405c6795d70f17b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:27,701 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,706 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,709 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:27,716 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:27,718 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 29997fa91edfab42405c6795d70f17b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72563224, jitterRate=0.0812762975692749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:28:27,725 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 29997fa91edfab42405c6795d70f17b3: Writing region info on filesystem at 1733286507693Initializing all the Stores at 1733286507695 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286507695Cleaning up temporary data from old regions at 1733286507706 (+11 ms)Region opened successfully at 1733286507724 (+18 ms) 2024-12-04T04:28:27,725 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 29997fa91edfab42405c6795d70f17b3, disabling compactions & flushes 2024-12-04T04:28:27,725 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. 2024-12-04T04:28:27,725 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. 2024-12-04T04:28:27,725 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. after waiting 0 ms 2024-12-04T04:28:27,725 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. 2024-12-04T04:28:27,726 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. 2024-12-04T04:28:27,726 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 29997fa91edfab42405c6795d70f17b3: Waiting for close lock at 1733286507725Disabling compacts and flushes for region at 1733286507725Disabling writes for close at 1733286507725Writing region close event to WAL at 1733286507726 (+1 ms)Closed at 1733286507726 2024-12-04T04:28:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741837_1013 (size=93) 2024-12-04T04:28:27,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741837_1013 (size=93) 2024-12-04T04:28:27,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741837_1013 (size=93) 2024-12-04T04:28:27,755 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:27,756 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21678556:(num 1733286507605) 2024-12-04T04:28:27,758 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T04:28:27,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741839_1015 (size=276) 2024-12-04T04:28:27,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741839_1015 (size=276) 2024-12-04T04:28:27,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741839_1015 (size=276) 2024-12-04T04:28:27,788 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T04:28:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741840_1016 (size=230) 2024-12-04T04:28:27,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741840_1016 (size=230) 2024-12-04T04:28:27,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741840_1016 (size=230) 2024-12-04T04:28:27,824 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1, size=276 (276bytes) 2024-12-04T04:28:27,825 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T04:28:27,825 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T04:28:27,825 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1 2024-12-04T04:28:27,830 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1 after 3ms 2024-12-04T04:28:27,835 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:27,836 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1 took 13ms 2024-12-04T04:28:27,840 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1 so closing down 2024-12-04T04:28:27,843 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T04:28:27,844 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:27,845 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T04:28:27,846 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:27,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741841_1017 (size=276) 2024-12-04T04:28:27,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741841_1017 (size=276) 2024-12-04T04:28:27,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741841_1017 (size=276) 2024-12-04T04:28:28,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:28,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002 2024-12-04T04:28:28,277 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 435 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T04:28:28,277 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1, journal: Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1, size=276 (276bytes) at 1733286507824Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1 so closing down at 1733286507840 (+16 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp at 1733286507845 (+5 ms)3 split writer threads finished at 1733286507846 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733286508265 (+419 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002 at 1733286508272 (+7 ms)Processed 2 edits across 1 Regions in 435 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733286508277 (+5 ms) 2024-12-04T04:28:28,293 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2, size=230 (230bytes) 2024-12-04T04:28:28,293 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2 2024-12-04T04:28:28,294 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2 after 1ms 2024-12-04T04:28:28,299 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:28,300 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2 took 7ms 2024-12-04T04:28:28,306 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2 so closing down 2024-12-04T04:28:28,306 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:28,314 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T04:28:28,318 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T04:28:28,318 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741842_1018 (size=230) 2024-12-04T04:28:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741842_1018 (size=230) 2024-12-04T04:28:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741842_1018 (size=230) 2024-12-04T04:28:28,739 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:28,750 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:28,755 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-04T04:28:28,758 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 454 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T04:28:28,758 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2, journal: Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2, size=230 (230bytes) at 1733286508293Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2 so closing down at 1733286508306 (+13 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002-wal-2.temp at 1733286508318 (+12 ms)3 split writer threads finished at 1733286508318Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733286508739 (+421 ms)Processed 1 edits across 1 Regions in 454 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733286508758 (+19 ms) 2024-12-04T04:28:28,758 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:28,762 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:28,778 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal.1733286508762, exclude list is [], retry=0 2024-12-04T04:28:28,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:28,790 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:28,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:28,800 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal.1733286508762 2024-12-04T04:28:28,804 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:28,805 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 29997fa91edfab42405c6795d70f17b3, NAME => 'testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:28,805 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:28,805 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,805 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,808 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,814 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29997fa91edfab42405c6795d70f17b3 columnFamilyName a 2024-12-04T04:28:28,815 DEBUG [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:28,816 INFO [StoreOpener-29997fa91edfab42405c6795d70f17b3-1 {}] regionserver.HStore(327): Store=29997fa91edfab42405c6795d70f17b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:28,816 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,824 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:28,826 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002 2024-12-04T04:28:28,832 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:28,845 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002 2024-12-04T04:28:28,852 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29997fa91edfab42405c6795d70f17b3 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T04:28:28,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/.tmp/a/afe92d13a56246279d9170255472ae00 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733286507756/Put/seqid=0 2024-12-04T04:28:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741844_1020 (size=5170) 2024-12-04T04:28:29,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741844_1020 (size=5170) 2024-12-04T04:28:29,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741844_1020 (size=5170) 2024-12-04T04:28:29,094 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/.tmp/a/afe92d13a56246279d9170255472ae00 2024-12-04T04:28:29,205 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/.tmp/a/afe92d13a56246279d9170255472ae00 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/a/afe92d13a56246279d9170255472ae00 2024-12-04T04:28:29,247 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/a/afe92d13a56246279d9170255472ae00, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T04:28:29,259 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 29997fa91edfab42405c6795d70f17b3 in 404ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T04:28:29,272 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/0000000000000000002 2024-12-04T04:28:29,284 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:29,284 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:29,312 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 29997fa91edfab42405c6795d70f17b3 2024-12-04T04:28:29,328 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/29997fa91edfab42405c6795d70f17b3/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T04:28:29,351 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 29997fa91edfab42405c6795d70f17b3; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69815693, jitterRate=0.04033489525318146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:28:29,352 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 29997fa91edfab42405c6795d70f17b3: Writing region info on filesystem at 1733286508805Initializing all the Stores at 1733286508807 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286508808 (+1 ms)Obtaining lock to block concurrent updates at 1733286508852 (+44 ms)Preparing flush snapshotting stores in 29997fa91edfab42405c6795d70f17b3 at 1733286508852Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733286508857 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733286507595.29997fa91edfab42405c6795d70f17b3. at 1733286508857Flushing 29997fa91edfab42405c6795d70f17b3/a: creating writer at 1733286508859 (+2 ms)Flushing 29997fa91edfab42405c6795d70f17b3/a: appending metadata at 1733286508980 (+121 ms)Flushing 29997fa91edfab42405c6795d70f17b3/a: closing flushed file at 1733286508984 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c8e8f: reopening flushed file at 1733286509192 (+208 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 29997fa91edfab42405c6795d70f17b3 in 404ms, sequenceid=2, compaction requested=false; wal=null at 1733286509259 (+67 ms)Cleaning up temporary data from old regions at 1733286509284 (+25 ms)Region opened successfully at 1733286509352 (+68 ms) 2024-12-04T04:28:29,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=371 (was 360) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39968 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:34962 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:56778 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:34906 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35021/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:56790 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39952 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=792 (was 792), ProcessCount=11 (was 11), AvailableMemoryMB=6295 (was 6743) 2024-12-04T04:28:29,433 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=371, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=792, ProcessCount=11, AvailableMemoryMB=6289 2024-12-04T04:28:29,456 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:29,463 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:29,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:29,490 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38759204, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-38759204, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:29,511 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38759204/hregion-38759204.1733286509491, exclude list is [], retry=0 2024-12-04T04:28:29,527 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:29,532 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:29,549 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:29,585 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38759204/hregion-38759204.1733286509491 2024-12-04T04:28:29,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:29,589 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 087d2d3d568f916fbbd32e52a57a427c, NAME => 'testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:29,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741846_1022 (size=64) 2024-12-04T04:28:29,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741846_1022 (size=64) 2024-12-04T04:28:29,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741846_1022 (size=64) 2024-12-04T04:28:29,705 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:29,723 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,729 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 087d2d3d568f916fbbd32e52a57a427c columnFamilyName a 2024-12-04T04:28:29,729 DEBUG [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:29,730 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] regionserver.HStore(327): Store=087d2d3d568f916fbbd32e52a57a427c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:29,731 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,738 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,744 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,744 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,764 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:29,792 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:29,794 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 087d2d3d568f916fbbd32e52a57a427c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73212513, jitterRate=0.09095145761966705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:28:29,800 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 087d2d3d568f916fbbd32e52a57a427c: Writing region info on filesystem at 1733286509705Initializing all the Stores at 1733286509720 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286509720Cleaning up temporary data from old regions at 1733286509744 (+24 ms)Region opened successfully at 1733286509795 (+51 ms) 2024-12-04T04:28:29,800 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 087d2d3d568f916fbbd32e52a57a427c, disabling compactions & flushes 2024-12-04T04:28:29,800 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. 2024-12-04T04:28:29,800 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. 2024-12-04T04:28:29,801 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. after waiting 0 ms 2024-12-04T04:28:29,801 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. 2024-12-04T04:28:29,808 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. 2024-12-04T04:28:29,808 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 087d2d3d568f916fbbd32e52a57a427c: Waiting for close lock at 1733286509800Disabling compacts and flushes for region at 1733286509800Disabling writes for close at 1733286509801 (+1 ms)Writing region close event to WAL at 1733286509808 (+7 ms)Closed at 1733286509808 2024-12-04T04:28:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741845_1021 (size=93) 2024-12-04T04:28:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741845_1021 (size=93) 2024-12-04T04:28:29,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741845_1021 (size=93) 2024-12-04T04:28:29,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:29,853 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38759204:(num 1733286509491) 2024-12-04T04:28:29,857 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T04:28:29,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741847_1023 (size=276) 2024-12-04T04:28:29,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741847_1023 (size=276) 2024-12-04T04:28:29,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741847_1023 (size=276) 2024-12-04T04:28:29,969 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T04:28:30,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741848_1024 (size=230) 2024-12-04T04:28:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741848_1024 (size=230) 2024-12-04T04:28:30,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741848_1024 (size=230) 2024-12-04T04:28:30,094 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2, size=230 (230bytes) 2024-12-04T04:28:30,095 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2 2024-12-04T04:28:30,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2 after 5ms 2024-12-04T04:28:30,113 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:30,113 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2 took 19ms 2024-12-04T04:28:30,144 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2 so closing down 2024-12-04T04:28:30,145 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:30,160 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T04:28:30,171 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T04:28:30,171 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:30,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741849_1025 (size=230) 2024-12-04T04:28:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741849_1025 (size=230) 2024-12-04T04:28:30,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741849_1025 (size=230) 2024-12-04T04:28:30,244 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:30,259 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 2024-12-04T04:28:30,268 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 132 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T04:28:30,268 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2, journal: Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2, size=230 (230bytes) at 1733286510094Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2 so closing down at 1733286510144 (+50 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp at 1733286510171 (+27 ms)3 split writer threads finished at 1733286510171Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733286510244 (+73 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 at 1733286510259 (+15 ms)Processed 1 edits across 1 Regions in 132 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733286510268 (+9 ms) 2024-12-04T04:28:30,295 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1, size=276 (276bytes) 2024-12-04T04:28:30,295 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1 2024-12-04T04:28:30,297 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1 after 2ms 2024-12-04T04:28:30,310 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:30,310 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1 took 15ms 2024-12-04T04:28:30,341 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1 so closing down 2024-12-04T04:28:30,341 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:30,348 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T04:28:30,351 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T04:28:30,351 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:30,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741850_1026 (size=276) 2024-12-04T04:28:30,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741850_1026 (size=276) 2024-12-04T04:28:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741850_1026 (size=276) 2024-12-04T04:28:30,384 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:30,407 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:30,413 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002, length=230 2024-12-04T04:28:30,418 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 2024-12-04T04:28:30,419 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T04:28:30,419 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1, journal: Splitting hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1, size=276 (276bytes) at 1733286510295Finishing writing output for hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1 so closing down at 1733286510341 (+46 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp at 1733286510351 (+10 ms)3 split writer threads finished at 1733286510351Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733286510384 (+33 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 at 1733286510419 (+35 ms)Processed 2 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733286510419 2024-12-04T04:28:30,419 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:30,426 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:30,444 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal.1733286510427, exclude list is [], retry=0 2024-12-04T04:28:30,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:30,453 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:30,453 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:30,461 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal.1733286510427 2024-12-04T04:28:30,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:30,461 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 087d2d3d568f916fbbd32e52a57a427c, NAME => 'testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,464 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,472 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 087d2d3d568f916fbbd32e52a57a427c columnFamilyName a 2024-12-04T04:28:30,473 DEBUG [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:30,474 INFO [StoreOpener-087d2d3d568f916fbbd32e52a57a427c-1 {}] regionserver.HStore(327): Store=087d2d3d568f916fbbd32e52a57a427c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:30,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,484 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 2024-12-04T04:28:30,497 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:30,502 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 2024-12-04T04:28:30,502 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 087d2d3d568f916fbbd32e52a57a427c 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T04:28:30,546 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/.tmp/a/66ee8682f51c482ab464b10e68a69929 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733286509853/Put/seqid=0 2024-12-04T04:28:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741852_1028 (size=5170) 2024-12-04T04:28:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741852_1028 (size=5170) 2024-12-04T04:28:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741852_1028 (size=5170) 2024-12-04T04:28:30,595 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/.tmp/a/66ee8682f51c482ab464b10e68a69929 2024-12-04T04:28:30,628 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/.tmp/a/66ee8682f51c482ab464b10e68a69929 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/a/66ee8682f51c482ab464b10e68a69929 2024-12-04T04:28:30,641 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/a/66ee8682f51c482ab464b10e68a69929, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T04:28:30,642 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 087d2d3d568f916fbbd32e52a57a427c in 140ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T04:28:30,643 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/0000000000000000002 2024-12-04T04:28:30,645 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,645 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,649 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 087d2d3d568f916fbbd32e52a57a427c 2024-12-04T04:28:30,657 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/087d2d3d568f916fbbd32e52a57a427c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T04:28:30,670 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 087d2d3d568f916fbbd32e52a57a427c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71535891, jitterRate=0.06596784293651581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:28:30,670 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 087d2d3d568f916fbbd32e52a57a427c: Writing region info on filesystem at 1733286510462Initializing all the Stores at 1733286510463 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286510463Obtaining lock to block concurrent updates at 1733286510502 (+39 ms)Preparing flush snapshotting stores in 087d2d3d568f916fbbd32e52a57a427c at 1733286510502Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733286510503 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733286509457.087d2d3d568f916fbbd32e52a57a427c. at 1733286510503Flushing 087d2d3d568f916fbbd32e52a57a427c/a: creating writer at 1733286510503Flushing 087d2d3d568f916fbbd32e52a57a427c/a: appending metadata at 1733286510545 (+42 ms)Flushing 087d2d3d568f916fbbd32e52a57a427c/a: closing flushed file at 1733286510545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cd40c51: reopening flushed file at 1733286510626 (+81 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 087d2d3d568f916fbbd32e52a57a427c in 140ms, sequenceid=2, compaction requested=false; wal=null at 1733286510642 (+16 ms)Cleaning up temporary data from old regions at 1733286510645 (+3 ms)Region opened successfully at 1733286510670 (+25 ms) 2024-12-04T04:28:30,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=381 (was 371) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:40076 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:35032 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:56778 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:56834 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39952 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=792 (was 792), ProcessCount=11 (was 11), AvailableMemoryMB=6119 (was 6289) 2024-12-04T04:28:30,711 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=381, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=792, ProcessCount=11, AvailableMemoryMB=6117 2024-12-04T04:28:30,734 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:30,739 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:30,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:30,765 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-14862694, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-14862694, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:30,782 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-14862694/hregion-14862694.1733286510765, exclude list is [], retry=0 2024-12-04T04:28:30,791 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:30,792 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:30,801 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:30,826 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-14862694/hregion-14862694.1733286510765 2024-12-04T04:28:30,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:30,828 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 54aea49057b9e5c2ebe347c9bcf1b92b, NAME => 'testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:30,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741854_1030 (size=64) 2024-12-04T04:28:30,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741854_1030 (size=64) 2024-12-04T04:28:30,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741854_1030 (size=64) 2024-12-04T04:28:30,929 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:30,940 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:30,950 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName a 2024-12-04T04:28:30,950 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:30,953 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:30,953 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:30,964 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName b 2024-12-04T04:28:30,964 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:30,972 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:30,972 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:30,988 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName c 2024-12-04T04:28:30,988 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:30,991 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:30,992 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:30,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:31,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:31,016 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:31,016 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:31,024 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:31,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:31,038 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:31,039 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 54aea49057b9e5c2ebe347c9bcf1b92b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70115202, jitterRate=0.044797927141189575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:31,040 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 54aea49057b9e5c2ebe347c9bcf1b92b: Writing region info on filesystem at 1733286510929Initializing all the Stores at 1733286510936 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286510936Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286510940 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286510940Cleaning up temporary data from old regions at 1733286511016 (+76 ms)Region opened successfully at 1733286511040 (+24 ms) 2024-12-04T04:28:31,040 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 54aea49057b9e5c2ebe347c9bcf1b92b, disabling compactions & flushes 2024-12-04T04:28:31,040 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:31,040 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:31,040 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. after waiting 0 ms 2024-12-04T04:28:31,040 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:31,052 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:31,052 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 54aea49057b9e5c2ebe347c9bcf1b92b: Waiting for close lock at 1733286511040Disabling compacts and flushes for region at 1733286511040Disabling writes for close at 1733286511040Writing region close event to WAL at 1733286511052 (+12 ms)Closed at 1733286511052 2024-12-04T04:28:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741853_1029 (size=93) 2024-12-04T04:28:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741853_1029 (size=93) 2024-12-04T04:28:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741853_1029 (size=93) 2024-12-04T04:28:31,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:31,089 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-14862694:(num 1733286510765) 2024-12-04T04:28:31,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:31,101 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:31,118 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, exclude list is [], retry=0 2024-12-04T04:28:31,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:31,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:31,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:31,137 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 2024-12-04T04:28:31,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:31,468 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, size=0 (0bytes) 2024-12-04T04:28:31,468 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 might be still open, length is 0 2024-12-04T04:28:31,468 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 2024-12-04T04:28:31,480 WARN [IPC Server handler 3 on default port 35021 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-04T04:28:31,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 after 14ms 2024-12-04T04:28:32,381 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T04:28:32,453 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T04:28:32,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T04:28:32,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:32,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T04:28:32,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T04:28:32,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T04:28:32,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:32,460 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-04T04:28:32,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:32,676 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:56876 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40387:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56876 dst: /127.0.0.1:40387 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40387 remote=/127.0.0.1:56876]. Total timeout mills is 60000, 58739 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:32,681 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:40104 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40104 dst: /127.0.0.1:43031 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:32,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:35068 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35068 dst: /127.0.0.1:38591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:32,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741855_1032 (size=200589) 2024-12-04T04:28:32,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741855_1032 (size=200589) 2024-12-04T04:28:32,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741855_1032 (size=200589) 2024-12-04T04:28:35,483 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 after 4015ms 2024-12-04T04:28:35,487 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:35,489 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 took 4021ms 2024-12-04T04:28:35,499 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733286511101.temp 2024-12-04T04:28:35,504 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp 2024-12-04T04:28:35,623 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101; continuing. 2024-12-04T04:28:35,623 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 so closing down 2024-12-04T04:28:35,623 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:35,623 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:35,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741856_1033 (size=200597) 2024-12-04T04:28:35,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741856_1033 (size=200597) 2024-12-04T04:28:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741856_1033 (size=200597) 2024-12-04T04:28:35,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp (wrote 3002 edits, skipped 0 edits in 37 ms) 2024-12-04T04:28:35,646 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 2024-12-04T04:28:35,646 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 154 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T04:28:35,646 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, journal: Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, size=0 (0bytes) at 1733286511468Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp at 1733286515504 (+4036 ms)Split 1024 edits, skipped 0 edits. at 1733286515561 (+57 ms)Split 2048 edits, skipped 0 edits. at 1733286515595 (+34 ms)Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 so closing down at 1733286515623 (+28 ms)3 split writer threads finished at 1733286515623Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp (wrote 3002 edits, skipped 0 edits in 37 ms) at 1733286515644 (+21 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000000001-wal.1733286511101.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 at 1733286515646 (+2 ms)Processed 3002 edits across 1 Regions in 154 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101, size=0, length=0, corrupted=false, cancelled=false at 1733286515646 2024-12-04T04:28:35,648 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286511101 2024-12-04T04:28:35,650 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 2024-12-04T04:28:35,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:35,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:35,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286515654, exclude list is [], retry=0 2024-12-04T04:28:35,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:35,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:35,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:35,683 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286515654 2024-12-04T04:28:35,683 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:35,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:35,685 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,686 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName a 2024-12-04T04:28:35,687 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:35,687 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:35,687 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,688 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName b 2024-12-04T04:28:35,688 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:35,689 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:35,689 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,691 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54aea49057b9e5c2ebe347c9bcf1b92b columnFamilyName c 2024-12-04T04:28:35,691 DEBUG [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:35,691 INFO [StoreOpener-54aea49057b9e5c2ebe347c9bcf1b92b-1 {}] regionserver.HStore(327): Store=54aea49057b9e5c2ebe347c9bcf1b92b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:35,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:35,696 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 2024-12-04T04:28:35,701 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:35,754 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T04:28:36,136 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54aea49057b9e5c2ebe347c9bcf1b92b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T04:28:36,170 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/61b1c06471a743c1846b2e0f0b06342c is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733286511149/Put/seqid=0 2024-12-04T04:28:36,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741858_1035 (size=50463) 2024-12-04T04:28:36,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741858_1035 (size=50463) 2024-12-04T04:28:36,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741858_1035 (size=50463) 2024-12-04T04:28:36,191 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/61b1c06471a743c1846b2e0f0b06342c 2024-12-04T04:28:36,200 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/61b1c06471a743c1846b2e0f0b06342c as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/a/61b1c06471a743c1846b2e0f0b06342c 2024-12-04T04:28:36,207 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/a/61b1c06471a743c1846b2e0f0b06342c, entries=754, sequenceid=754, filesize=49.3 K 2024-12-04T04:28:36,208 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 54aea49057b9e5c2ebe347c9bcf1b92b in 71ms, sequenceid=754, compaction requested=false; wal=null 2024-12-04T04:28:36,225 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T04:28:36,225 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54aea49057b9e5c2ebe347c9bcf1b92b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T04:28:36,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/a6e403a25b9842258f975bf58dcb351b is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733286511223/Put/seqid=0 2024-12-04T04:28:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741859_1036 (size=20072) 2024-12-04T04:28:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741859_1036 (size=20072) 2024-12-04T04:28:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741859_1036 (size=20072) 2024-12-04T04:28:36,247 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/a6e403a25b9842258f975bf58dcb351b 2024-12-04T04:28:36,273 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/8260c051815f4a79830ba3cd2beefc69 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733286511258/Put/seqid=0 2024-12-04T04:28:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741860_1037 (size=35835) 2024-12-04T04:28:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741860_1037 (size=35835) 2024-12-04T04:28:36,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741860_1037 (size=35835) 2024-12-04T04:28:36,286 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/8260c051815f4a79830ba3cd2beefc69 2024-12-04T04:28:36,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/a/a6e403a25b9842258f975bf58dcb351b as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/a/a6e403a25b9842258f975bf58dcb351b 2024-12-04T04:28:36,298 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/a/a6e403a25b9842258f975bf58dcb351b, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-04T04:28:36,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/8260c051815f4a79830ba3cd2beefc69 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/b/8260c051815f4a79830ba3cd2beefc69 2024-12-04T04:28:36,306 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/b/8260c051815f4a79830ba3cd2beefc69, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-04T04:28:36,306 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 54aea49057b9e5c2ebe347c9bcf1b92b in 81ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-04T04:28:36,317 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T04:28:36,317 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54aea49057b9e5c2ebe347c9bcf1b92b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T04:28:36,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/4535656d948c468597dcf8d1517fddd8 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733286511291/Put/seqid=0 2024-12-04T04:28:36,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741861_1038 (size=35082) 2024-12-04T04:28:36,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741861_1038 (size=35082) 2024-12-04T04:28:36,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741861_1038 (size=35082) 2024-12-04T04:28:36,336 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/4535656d948c468597dcf8d1517fddd8 2024-12-04T04:28:36,357 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/817e174f70744f189678151a36a82aec is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733286511342/Put/seqid=0 2024-12-04T04:28:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741862_1039 (size=20825) 2024-12-04T04:28:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741862_1039 (size=20825) 2024-12-04T04:28:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741862_1039 (size=20825) 2024-12-04T04:28:36,367 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/817e174f70744f189678151a36a82aec 2024-12-04T04:28:36,374 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/b/4535656d948c468597dcf8d1517fddd8 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/b/4535656d948c468597dcf8d1517fddd8 2024-12-04T04:28:36,380 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/b/4535656d948c468597dcf8d1517fddd8, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-04T04:28:36,381 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/817e174f70744f189678151a36a82aec as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/c/817e174f70744f189678151a36a82aec 2024-12-04T04:28:36,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/c/817e174f70744f189678151a36a82aec, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-04T04:28:36,388 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 54aea49057b9e5c2ebe347c9bcf1b92b in 70ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-04T04:28:36,396 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733286511411/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:36,399 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 2024-12-04T04:28:36,400 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T04:28:36,400 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54aea49057b9e5c2ebe347c9bcf1b92b 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-04T04:28:36,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/d5cf5d9a45bd4a318d57f0167a53555d is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733286511352/Put/seqid=0 2024-12-04T04:28:36,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741863_1040 (size=50301) 2024-12-04T04:28:36,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741863_1040 (size=50301) 2024-12-04T04:28:36,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741863_1040 (size=50301) 2024-12-04T04:28:36,420 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/d5cf5d9a45bd4a318d57f0167a53555d 2024-12-04T04:28:36,426 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d5cf5d9a45bd4a318d57f0167a53555d 2024-12-04T04:28:36,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/.tmp/c/d5cf5d9a45bd4a318d57f0167a53555d as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/c/d5cf5d9a45bd4a318d57f0167a53555d 2024-12-04T04:28:36,434 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d5cf5d9a45bd4a318d57f0167a53555d 2024-12-04T04:28:36,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/c/d5cf5d9a45bd4a318d57f0167a53555d, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-04T04:28:36,434 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 54aea49057b9e5c2ebe347c9bcf1b92b in 34ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-04T04:28:36,435 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/0000000000000003002 2024-12-04T04:28:36,436 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:36,436 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:36,437 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T04:28:36,438 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 54aea49057b9e5c2ebe347c9bcf1b92b 2024-12-04T04:28:36,441 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenIntoWAL/54aea49057b9e5c2ebe347c9bcf1b92b/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-04T04:28:36,442 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 54aea49057b9e5c2ebe347c9bcf1b92b; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59671572, jitterRate=-0.11082428693771362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T04:28:36,442 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 54aea49057b9e5c2ebe347c9bcf1b92b: Writing region info on filesystem at 1733286515683Initializing all the Stores at 1733286515685 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286515685Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286515685Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286515685Cleaning up temporary data from old regions at 1733286516436 (+751 ms)Region opened successfully at 1733286516442 (+6 ms) 2024-12-04T04:28:36,506 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 54aea49057b9e5c2ebe347c9bcf1b92b, disabling compactions & flushes 2024-12-04T04:28:36,507 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:36,507 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:36,507 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. after waiting 0 ms 2024-12-04T04:28:36,507 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:36,513 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733286510735.54aea49057b9e5c2ebe347c9bcf1b92b. 2024-12-04T04:28:36,513 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 54aea49057b9e5c2ebe347c9bcf1b92b: Waiting for close lock at 1733286516506Disabling compacts and flushes for region at 1733286516506Disabling writes for close at 1733286516507 (+1 ms)Writing region close event to WAL at 1733286516513 (+6 ms)Closed at 1733286516513 2024-12-04T04:28:36,518 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286515654 not finished, retry = 0 2024-12-04T04:28:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741857_1034 (size=93) 2024-12-04T04:28:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741857_1034 (size=93) 2024-12-04T04:28:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741857_1034 (size=93) 2024-12-04T04:28:36,622 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:36,622 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733286515654) 2024-12-04T04:28:36,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=392 (was 381) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_392146419_22 at /127.0.0.1:37374 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:46445 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35021 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46445 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:43743 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43743 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_392146419_22 at /127.0.0.1:54074 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_392146419_22 at /127.0.0.1:50630 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:35021 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_392146419_22 at /127.0.0.1:37346 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=861 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=791 (was 792), ProcessCount=11 (was 11), AvailableMemoryMB=5693 (was 6117) 2024-12-04T04:28:36,647 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=392, OpenFileDescriptor=861, MaxFileDescriptor=1048576, SystemLoadAverage=791, ProcessCount=11, AvailableMemoryMB=5690 2024-12-04T04:28:36,662 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:36,665 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:36,666 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:36,669 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08540230, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-08540230, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:36,681 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08540230/hregion-08540230.1733286516670, exclude list is [], retry=0 2024-12-04T04:28:36,684 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:36,684 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:36,684 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:36,690 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08540230/hregion-08540230.1733286516670 2024-12-04T04:28:36,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:36,691 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e7ef9d4d8cf80a1b8745631df2195ecd, NAME => 'test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741865_1042 (size=43) 2024-12-04T04:28:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741865_1042 (size=43) 2024-12-04T04:28:36,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741865_1042 (size=43) 2024-12-04T04:28:36,703 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:36,705 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,707 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName a 2024-12-04T04:28:36,707 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:36,708 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:36,708 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,709 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName b 2024-12-04T04:28:36,710 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:36,710 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:36,710 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,712 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName c 2024-12-04T04:28:36,712 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:36,713 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:36,713 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,716 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:36,718 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:36,724 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:36,725 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ef9d4d8cf80a1b8745631df2195ecd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71889752, jitterRate=0.07124078273773193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:36,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ef9d4d8cf80a1b8745631df2195ecd: Writing region info on filesystem at 1733286516703Initializing all the Stores at 1733286516705 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286516705Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286516705Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286516705Cleaning up temporary data from old regions at 1733286516716 (+11 ms)Region opened successfully at 1733286516725 (+9 ms) 2024-12-04T04:28:36,726 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e7ef9d4d8cf80a1b8745631df2195ecd, disabling compactions & flushes 2024-12-04T04:28:36,726 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:36,726 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:36,726 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. after waiting 0 ms 2024-12-04T04:28:36,726 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:36,726 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:36,727 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e7ef9d4d8cf80a1b8745631df2195ecd: Waiting for close lock at 1733286516726Disabling compacts and flushes for region at 1733286516726Disabling writes for close at 1733286516726Writing region close event to WAL at 1733286516726Closed at 1733286516726 2024-12-04T04:28:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741864_1041 (size=93) 2024-12-04T04:28:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741864_1041 (size=93) 2024-12-04T04:28:36,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741864_1041 (size=93) 2024-12-04T04:28:36,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:36,733 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08540230:(num 1733286516670) 2024-12-04T04:28:36,733 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:36,735 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:36,753 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, exclude list is [], retry=0 2024-12-04T04:28:36,756 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:36,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:36,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:36,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 2024-12-04T04:28:36,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:28:36,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741866_1043 (size=200357) 2024-12-04T04:28:36,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741866_1043 (size=200357) 2024-12-04T04:28:36,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741866_1043 (size=200357) 2024-12-04T04:28:36,989 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, size=195.7 K (200357bytes) 2024-12-04T04:28:36,989 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 2024-12-04T04:28:36,989 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 after 0ms 2024-12-04T04:28:36,993 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:36,994 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 took 6ms 2024-12-04T04:28:37,001 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733286516735.temp 2024-12-04T04:28:37,003 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp 2024-12-04T04:28:37,039 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 so closing down 2024-12-04T04:28:37,039 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:37,040 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:37,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741867_1044 (size=200357) 2024-12-04T04:28:37,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741867_1044 (size=200357) 2024-12-04T04:28:37,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741867_1044 (size=200357) 2024-12-04T04:28:37,049 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-12-04T04:28:37,052 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp to hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 2024-12-04T04:28:37,052 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-04T04:28:37,052 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, journal: Splitting hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, size=195.7 K (200357bytes) at 1733286516989Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp at 1733286517003 (+14 ms)Split 1024 edits, skipped 0 edits. at 1733286517009 (+6 ms)Split 2048 edits, skipped 0 edits. at 1733286517025 (+16 ms)Finishing writing output for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 so closing down at 1733286517039 (+14 ms)3 split writer threads finished at 1733286517040 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1733286517049 (+9 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000000001-wal.1733286516735.temp to hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 at 1733286517052 (+3 ms)Processed 3000 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733286517052 2024-12-04T04:28:37,055 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286516735 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286516735 2024-12-04T04:28:37,056 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 2024-12-04T04:28:37,056 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:37,061 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:37,080 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, exclude list is [], retry=0 2024-12-04T04:28:37,084 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:37,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:37,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:37,091 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 2024-12-04T04:28:37,091 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:37,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741868_1045 (size=200484) 2024-12-04T04:28:37,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741868_1045 (size=200484) 2024-12-04T04:28:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741868_1045 (size=200484) 2024-12-04T04:28:37,310 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, size=195.8 K (200484bytes) 2024-12-04T04:28:37,310 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 2024-12-04T04:28:37,310 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 after 0ms 2024-12-04T04:28:37,313 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:37,315 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 took 5ms 2024-12-04T04:28:37,320 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733286517061.temp 2024-12-04T04:28:37,322 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp 2024-12-04T04:28:37,369 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 so closing down 2024-12-04T04:28:37,369 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:37,370 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:37,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741869_1046 (size=200484) 2024-12-04T04:28:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741869_1046 (size=200484) 2024-12-04T04:28:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741869_1046 (size=200484) 2024-12-04T04:28:37,418 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-12-04T04:28:37,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp to hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 2024-12-04T04:28:37,421 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-04T04:28:37,421 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, journal: Splitting hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, size=195.8 K (200484bytes) at 1733286517310Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp at 1733286517322 (+12 ms)Split 1024 edits, skipped 0 edits. at 1733286517333 (+11 ms)Split 2048 edits, skipped 0 edits. at 1733286517348 (+15 ms)Finishing writing output for hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 so closing down at 1733286517369 (+21 ms)3 split writer threads finished at 1733286517370 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1733286517418 (+48 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003001-wal.1733286517061.temp to hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 at 1733286517421 (+3 ms)Processed 3000 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733286517421 2024-12-04T04:28:37,426 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517061 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286517061 2024-12-04T04:28:37,428 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 2024-12-04T04:28:37,428 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:37,432 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/test2727-manual,16010,1733286516662, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:37,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517433, exclude list is [], retry=0 2024-12-04T04:28:37,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:37,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:37,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:37,460 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733286516662/wal.1733286517433 2024-12-04T04:28:37,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:28:37,461 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e7ef9d4d8cf80a1b8745631df2195ecd, NAME => 'test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:37,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:37,461 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,461 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,463 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,464 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName a 2024-12-04T04:28:37,464 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:37,465 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:37,465 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,466 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName b 2024-12-04T04:28:37,466 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:37,467 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:37,467 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,469 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ef9d4d8cf80a1b8745631df2195ecd columnFamilyName c 2024-12-04T04:28:37,469 DEBUG [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:37,470 INFO [StoreOpener-e7ef9d4d8cf80a1b8745631df2195ecd-1 {}] regionserver.HStore(327): Store=e7ef9d4d8cf80a1b8745631df2195ecd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:37,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,474 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,475 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 2024-12-04T04:28:37,478 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:37,519 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 2024-12-04T04:28:37,520 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 2024-12-04T04:28:37,523 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:37,574 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 2024-12-04T04:28:37,574 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7ef9d4d8cf80a1b8745631df2195ecd 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-04T04:28:37,597 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/a/c4e0a7657ca543b7aacbb48e98fd8a2b is 41, key is test2727/a:100/1733286517098/Put/seqid=0 2024-12-04T04:28:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741871_1048 (size=84227) 2024-12-04T04:28:37,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741871_1048 (size=84227) 2024-12-04T04:28:37,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741871_1048 (size=84227) 2024-12-04T04:28:37,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/a/c4e0a7657ca543b7aacbb48e98fd8a2b 2024-12-04T04:28:37,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/b/e7a7bea3a4164608ac6d54d1c75742c6 is 41, key is test2727/b:100/1733286517165/Put/seqid=0 2024-12-04T04:28:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741872_1049 (size=84609) 2024-12-04T04:28:37,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741872_1049 (size=84609) 2024-12-04T04:28:37,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741872_1049 (size=84609) 2024-12-04T04:28:37,670 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/b/e7a7bea3a4164608ac6d54d1c75742c6 2024-12-04T04:28:37,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/c/4f5c0f381e2743da841d03d028db034e is 41, key is test2727/c:100/1733286517229/Put/seqid=0 2024-12-04T04:28:37,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741873_1050 (size=84609) 2024-12-04T04:28:37,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741873_1050 (size=84609) 2024-12-04T04:28:37,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741873_1050 (size=84609) 2024-12-04T04:28:37,723 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/c/4f5c0f381e2743da841d03d028db034e 2024-12-04T04:28:37,735 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/a/c4e0a7657ca543b7aacbb48e98fd8a2b as hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/a/c4e0a7657ca543b7aacbb48e98fd8a2b 2024-12-04T04:28:37,743 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/a/c4e0a7657ca543b7aacbb48e98fd8a2b, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-04T04:28:37,744 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/b/e7a7bea3a4164608ac6d54d1c75742c6 as hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/b/e7a7bea3a4164608ac6d54d1c75742c6 2024-12-04T04:28:37,752 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/b/e7a7bea3a4164608ac6d54d1c75742c6, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T04:28:37,754 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/.tmp/c/4f5c0f381e2743da841d03d028db034e as hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/c/4f5c0f381e2743da841d03d028db034e 2024-12-04T04:28:37,761 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/c/4f5c0f381e2743da841d03d028db034e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T04:28:37,761 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e7ef9d4d8cf80a1b8745631df2195ecd in 187ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-04T04:28:37,762 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000003000 2024-12-04T04:28:37,763 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/0000000000000006000 2024-12-04T04:28:37,764 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,764 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,765 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:37,767 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ef9d4d8cf80a1b8745631df2195ecd 2024-12-04T04:28:37,769 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/test2727/e7ef9d4d8cf80a1b8745631df2195ecd/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-04T04:28:37,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ef9d4d8cf80a1b8745631df2195ecd; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62783775, jitterRate=-0.06444884836673737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:37,771 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ef9d4d8cf80a1b8745631df2195ecd: Writing region info on filesystem at 1733286517461Initializing all the Stores at 1733286517463 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286517463Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286517463Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286517463Obtaining lock to block concurrent updates at 1733286517574 (+111 ms)Preparing flush snapshotting stores in e7ef9d4d8cf80a1b8745631df2195ecd at 1733286517574Finished memstore snapshotting test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733286517574Flushing stores of test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. at 1733286517574Flushing e7ef9d4d8cf80a1b8745631df2195ecd/a: creating writer at 1733286517574Flushing e7ef9d4d8cf80a1b8745631df2195ecd/a: appending metadata at 1733286517596 (+22 ms)Flushing e7ef9d4d8cf80a1b8745631df2195ecd/a: closing flushed file at 1733286517596Flushing e7ef9d4d8cf80a1b8745631df2195ecd/b: creating writer at 1733286517621 (+25 ms)Flushing e7ef9d4d8cf80a1b8745631df2195ecd/b: appending metadata at 1733286517649 (+28 ms)Flushing e7ef9d4d8cf80a1b8745631df2195ecd/b: closing flushed file at 1733286517649Flushing e7ef9d4d8cf80a1b8745631df2195ecd/c: creating writer at 1733286517679 (+30 ms)Flushing e7ef9d4d8cf80a1b8745631df2195ecd/c: appending metadata at 1733286517705 (+26 ms)Flushing e7ef9d4d8cf80a1b8745631df2195ecd/c: closing flushed file at 1733286517705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@431b1ee: reopening flushed file at 1733286517734 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fc846e: reopening flushed file at 1733286517743 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75595629: reopening flushed file at 1733286517752 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e7ef9d4d8cf80a1b8745631df2195ecd in 187ms, sequenceid=6000, compaction requested=false; wal=null at 1733286517761 (+9 ms)Cleaning up temporary data from old regions at 1733286517764 (+3 ms)Region opened successfully at 1733286517771 (+7 ms) 2024-12-04T04:28:37,772 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-04T04:28:37,773 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e7ef9d4d8cf80a1b8745631df2195ecd, disabling compactions & flushes 2024-12-04T04:28:37,773 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:37,773 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:37,773 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. after waiting 0 ms 2024-12-04T04:28:37,773 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:37,774 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733286516663.e7ef9d4d8cf80a1b8745631df2195ecd. 2024-12-04T04:28:37,774 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e7ef9d4d8cf80a1b8745631df2195ecd: Waiting for close lock at 1733286517772Disabling compacts and flushes for region at 1733286517772Disabling writes for close at 1733286517773 (+1 ms)Writing region close event to WAL at 1733286517774 (+1 ms)Closed at 1733286517774 2024-12-04T04:28:37,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741870_1047 (size=93) 2024-12-04T04:28:37,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741870_1047 (size=93) 2024-12-04T04:28:37,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741870_1047 (size=93) 2024-12-04T04:28:37,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:37,780 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733286517433) 2024-12-04T04:28:37,792 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=392 (was 392), OpenFileDescriptor=921 (was 861) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=791 (was 791), ProcessCount=11 (was 11), AvailableMemoryMB=5428 (was 5690) 2024-12-04T04:28:37,803 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=392, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=791, ProcessCount=11, AvailableMemoryMB=5425 2024-12-04T04:28:37,816 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:37,822 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:37,823 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733286517823 2024-12-04T04:28:37,833 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 2024-12-04T04:28:37,835 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:37,836 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c3ac9d10a394f55a8789ca35d0c5521b, NAME => 'testSequentialEditLogSeqNum,,1733286517817.c3ac9d10a394f55a8789ca35d0c5521b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:37,836 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733286517817.c3ac9d10a394f55a8789ca35d0c5521b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:37,837 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,837 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,838 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b doesn't exist for region: c3ac9d10a394f55a8789ca35d0c5521b on table testSequentialEditLogSeqNum 2024-12-04T04:28:37,838 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: c3ac9d10a394f55a8789ca35d0c5521b on table testSequentialEditLogSeqNum 2024-12-04T04:28:37,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741875_1052 (size=62) 2024-12-04T04:28:37,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741875_1052 (size=62) 2024-12-04T04:28:37,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741875_1052 (size=62) 2024-12-04T04:28:37,849 INFO [StoreOpener-c3ac9d10a394f55a8789ca35d0c5521b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,851 INFO [StoreOpener-c3ac9d10a394f55a8789ca35d0c5521b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ac9d10a394f55a8789ca35d0c5521b columnFamilyName a 2024-12-04T04:28:37,851 DEBUG [StoreOpener-c3ac9d10a394f55a8789ca35d0c5521b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:37,852 INFO [StoreOpener-c3ac9d10a394f55a8789ca35d0c5521b-1 {}] regionserver.HStore(327): Store=c3ac9d10a394f55a8789ca35d0c5521b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:37,852 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,853 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,854 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,856 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c3ac9d10a394f55a8789ca35d0c5521b 2024-12-04T04:28:37,858 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:37,859 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c3ac9d10a394f55a8789ca35d0c5521b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64685964, jitterRate=-0.036104023456573486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:28:37,860 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c3ac9d10a394f55a8789ca35d0c5521b: Writing region info on filesystem at 1733286517837Initializing all the Stores at 1733286517848 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286517848Cleaning up temporary data from old regions at 1733286517854 (+6 ms)Region opened successfully at 1733286517860 (+6 ms) 2024-12-04T04:28:37,873 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c3ac9d10a394f55a8789ca35d0c5521b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-04T04:28:37,894 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/.tmp/a/7dd2d2423d014e718099b270dcbf5da5 is 81, key is testSequentialEditLogSeqNum/a:x0/1733286517860/Put/seqid=0 2024-12-04T04:28:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741876_1053 (size=5833) 2024-12-04T04:28:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741876_1053 (size=5833) 2024-12-04T04:28:37,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741876_1053 (size=5833) 2024-12-04T04:28:37,905 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/.tmp/a/7dd2d2423d014e718099b270dcbf5da5 2024-12-04T04:28:37,914 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/.tmp/a/7dd2d2423d014e718099b270dcbf5da5 as hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/a/7dd2d2423d014e718099b270dcbf5da5 2024-12-04T04:28:37,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/a/7dd2d2423d014e718099b270dcbf5da5, entries=10, sequenceid=13, filesize=5.7 K 2024-12-04T04:28:37,929 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c3ac9d10a394f55a8789ca35d0c5521b in 57ms, sequenceid=13, compaction requested=false 2024-12-04T04:28:37,929 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c3ac9d10a394f55a8789ca35d0c5521b: 2024-12-04T04:28:37,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T04:28:37,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T04:28:37,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T04:28:37,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T04:28:37,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T04:28:37,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741874_1051 (size=1616) 2024-12-04T04:28:37,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741874_1051 (size=1616) 2024-12-04T04:28:37,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741874_1051 (size=1616) 2024-12-04T04:28:37,971 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823, size=1.6 K (1616bytes) 2024-12-04T04:28:37,971 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 2024-12-04T04:28:37,972 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 after 1ms 2024-12-04T04:28:37,977 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:37,977 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 took 6ms 2024-12-04T04:28:37,983 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 so closing down 2024-12-04T04:28:37,983 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:37,985 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733286517823.temp 2024-12-04T04:28:37,996 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp 2024-12-04T04:28:37,999 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:38,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741877_1054 (size=1175) 2024-12-04T04:28:38,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741877_1054 (size=1175) 2024-12-04T04:28:38,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741877_1054 (size=1175) 2024-12-04T04:28:38,012 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp (wrote 15 edits, skipped 0 edits in 2 ms) 2024-12-04T04:28:38,014 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp to hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000020 2024-12-04T04:28:38,014 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-04T04:28:38,014 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823, journal: Splitting hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823, size=1.6 K (1616bytes) at 1733286517971Finishing writing output for hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823 so closing down at 1733286517983 (+12 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp at 1733286517996 (+13 ms)3 split writer threads finished at 1733286517999 (+3 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp (wrote 15 edits, skipped 0 edits in 2 ms) at 1733286518013 (+14 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000003-wal.1733286517823.temp to hdfs://localhost:35021/hbase/data/default/testSequentialEditLogSeqNum/c3ac9d10a394f55a8789ca35d0c5521b/recovered.edits/0000000000000000020 at 1733286518014 (+1 ms)Processed 17 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733286517816/wal.1733286517823, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733286518014 2024-12-04T04:28:38,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=397 (was 392) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:54178 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:37446 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:50774 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=791 (was 791), ProcessCount=11 (was 11), AvailableMemoryMB=5317 (was 5425) 2024-12-04T04:28:38,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=397, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=791, ProcessCount=11, AvailableMemoryMB=5314 2024-12-04T04:28:38,051 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:38,054 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:38,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:38,092 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-84261351, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-84261351, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:38,106 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-84261351/hregion-84261351.1733286518092, exclude list is [], retry=0 2024-12-04T04:28:38,110 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:38,110 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:38,110 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:38,115 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-84261351/hregion-84261351.1733286518092 2024-12-04T04:28:38,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:38,116 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9bfd3cf5b200456b326192fe6d0a9c5f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741879_1056 (size=70) 2024-12-04T04:28:38,138 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:38,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741879_1056 (size=70) 2024-12-04T04:28:38,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741879_1056 (size=70) 2024-12-04T04:28:38,144 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,147 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName a 2024-12-04T04:28:38,147 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,148 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,148 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,150 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName b 2024-12-04T04:28:38,150 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,151 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,151 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,153 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName c 2024-12-04T04:28:38,153 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,154 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,154 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,156 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,156 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,157 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:38,158 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,161 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:38,161 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bfd3cf5b200456b326192fe6d0a9c5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67796865, jitterRate=0.010252013802528381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:38,162 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bfd3cf5b200456b326192fe6d0a9c5f: Writing region info on filesystem at 1733286518138Initializing all the Stores at 1733286518143 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518143Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518144 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518144Cleaning up temporary data from old regions at 1733286518156 (+12 ms)Region opened successfully at 1733286518162 (+6 ms) 2024-12-04T04:28:38,162 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bfd3cf5b200456b326192fe6d0a9c5f, disabling compactions & flushes 2024-12-04T04:28:38,162 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:38,162 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:38,162 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. after waiting 0 ms 2024-12-04T04:28:38,162 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:38,163 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:38,163 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bfd3cf5b200456b326192fe6d0a9c5f: Waiting for close lock at 1733286518162Disabling compacts and flushes for region at 1733286518162Disabling writes for close at 1733286518162Writing region close event to WAL at 1733286518162Closed at 1733286518163 (+1 ms) 2024-12-04T04:28:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741878_1055 (size=93) 2024-12-04T04:28:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741878_1055 (size=93) 2024-12-04T04:28:38,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741878_1055 (size=93) 2024-12-04T04:28:38,175 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:38,175 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-84261351:(num 1733286518092) 2024-12-04T04:28:38,175 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:38,178 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:38,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, exclude list is [], retry=0 2024-12-04T04:28:38,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:38,197 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:38,197 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:38,200 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 2024-12-04T04:28:38,203 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:38,203 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bfd3cf5b200456b326192fe6d0a9c5f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,204 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,206 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,207 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName a 2024-12-04T04:28:38,207 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,207 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,207 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,208 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName b 2024-12-04T04:28:38,208 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,209 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,209 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,210 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName c 2024-12-04T04:28:38,210 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:38,211 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:38,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,216 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,217 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:38,218 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:38,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bfd3cf5b200456b326192fe6d0a9c5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71992542, jitterRate=0.07277247309684753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:38,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bfd3cf5b200456b326192fe6d0a9c5f: Writing region info on filesystem at 1733286518204Initializing all the Stores at 1733286518205 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518205Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518205Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286518205Cleaning up temporary data from old regions at 1733286518216 (+11 ms)Region opened successfully at 1733286518220 (+4 ms) 2024-12-04T04:28:38,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733286518223/Put/seqid=0 2024-12-04T04:28:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741881_1058 (size=4826) 2024-12-04T04:28:38,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741881_1058 (size=4826) 2024-12-04T04:28:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741881_1058 (size=4826) 2024-12-04T04:28:38,236 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35021/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 9bfd3cf5b200456b326192fe6d0a9c5f/a 2024-12-04T04:28:38,245 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-04T04:28:38,245 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T04:28:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9bfd3cf5b200456b326192fe6d0a9c5f: 2024-12-04T04:28:38,247 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/69e4f85b4c45438cbb964b78344f750a_SeqId_3_ 2024-12-04T04:28:38,248 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35021/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 9bfd3cf5b200456b326192fe6d0a9c5f/a as hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/69e4f85b4c45438cbb964b78344f750a_SeqId_3_ - updating store file list. 2024-12-04T04:28:38,256 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 69e4f85b4c45438cbb964b78344f750a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T04:28:38,257 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/69e4f85b4c45438cbb964b78344f750a_SeqId_3_ into 9bfd3cf5b200456b326192fe6d0a9c5f/a 2024-12-04T04:28:38,257 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35021/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 9bfd3cf5b200456b326192fe6d0a9c5f/a (new location: hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/69e4f85b4c45438cbb964b78344f750a_SeqId_3_) 2024-12-04T04:28:38,311 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, size=0 (0bytes) 2024-12-04T04:28:38,311 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 might be still open, length is 0 2024-12-04T04:28:38,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 2024-12-04T04:28:38,312 WARN [IPC Server handler 1 on default port 35021 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-04T04:28:38,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 after 1ms 2024-12-04T04:28:38,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:37548 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40387:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37548 dst: /127.0.0.1:40387 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40387 remote=/127.0.0.1:37548]. Total timeout mills is 60000, 59621 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:38,646 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:54252 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54252 dst: /127.0.0.1:43031 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:38,646 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:50882 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50882 dst: /127.0.0.1:38591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:38,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741880_1059 (size=434) 2024-12-04T04:28:38,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741880_1059 (size=434) 2024-12-04T04:28:42,313 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 after 4002ms 2024-12-04T04:28:42,316 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:42,317 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 took 4006ms 2024-12-04T04:28:42,319 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179; continuing. 2024-12-04T04:28:42,319 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 so closing down 2024-12-04T04:28:42,319 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:42,322 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733286518179.temp 2024-12-04T04:28:42,323 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp 2024-12-04T04:28:42,324 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741882_1060 (size=236) 2024-12-04T04:28:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741882_1060 (size=236) 2024-12-04T04:28:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741882_1060 (size=236) 2024-12-04T04:28:42,355 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:42,356 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp to hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 2024-12-04T04:28:42,356 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T04:28:42,356 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, journal: Splitting hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, size=0 (0bytes) at 1733286518311Finishing writing output for hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 so closing down at 1733286522319 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp at 1733286522323 (+4 ms)3 split writer threads finished at 1733286522324 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733286522355 (+31 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005-wal.1733286518179.temp to hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 at 1733286522356 (+1 ms)Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179, size=0, length=0, corrupted=false, cancelled=false at 1733286522356 2024-12-04T04:28:42,358 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286518179 2024-12-04T04:28:42,359 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 2024-12-04T04:28:42,359 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:42,361 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:42,363 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-04T04:28:42,364 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:42,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-04T04:28:42,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:42,373 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286522361, exclude list is [], retry=0 2024-12-04T04:28:42,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:42,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:42,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:42,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286522361 2024-12-04T04:28:42,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:42,379 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bfd3cf5b200456b326192fe6d0a9c5f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:42,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:42,379 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,379 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,381 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,382 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName a 2024-12-04T04:28:42,382 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,389 DEBUG [StoreFileOpener-9bfd3cf5b200456b326192fe6d0a9c5f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 69e4f85b4c45438cbb964b78344f750a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T04:28:42,389 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/69e4f85b4c45438cbb964b78344f750a_SeqId_3_ 2024-12-04T04:28:42,389 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:42,389 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,390 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName b 2024-12-04T04:28:42,391 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,391 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:42,392 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,393 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bfd3cf5b200456b326192fe6d0a9c5f columnFamilyName c 2024-12-04T04:28:42,393 DEBUG [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,394 INFO [StoreOpener-9bfd3cf5b200456b326192fe6d0a9c5f-1 {}] regionserver.HStore(327): Store=9bfd3cf5b200456b326192fe6d0a9c5f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:42,396 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,397 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,400 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 2024-12-04T04:28:42,403 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:42,405 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 2024-12-04T04:28:42,405 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9bfd3cf5b200456b326192fe6d0a9c5f 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-04T04:28:42,421 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/.tmp/a/47574299ae214a048bbee2f0a807a45d is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733286518265/Put/seqid=0 2024-12-04T04:28:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741884_1062 (size=5149) 2024-12-04T04:28:42,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741884_1062 (size=5149) 2024-12-04T04:28:42,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741884_1062 (size=5149) 2024-12-04T04:28:42,448 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/.tmp/a/47574299ae214a048bbee2f0a807a45d 2024-12-04T04:28:42,460 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/.tmp/a/47574299ae214a048bbee2f0a807a45d as hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/47574299ae214a048bbee2f0a807a45d 2024-12-04T04:28:42,467 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/a/47574299ae214a048bbee2f0a807a45d, entries=1, sequenceid=5, filesize=5.0 K 2024-12-04T04:28:42,467 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 9bfd3cf5b200456b326192fe6d0a9c5f in 62ms, sequenceid=5, compaction requested=false; wal=null 2024-12-04T04:28:42,468 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/0000000000000000005 2024-12-04T04:28:42,473 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,475 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:42,478 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bfd3cf5b200456b326192fe6d0a9c5f 2024-12-04T04:28:42,481 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/9bfd3cf5b200456b326192fe6d0a9c5f/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T04:28:42,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bfd3cf5b200456b326192fe6d0a9c5f; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63555818, jitterRate=-0.05294451117515564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:42,482 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bfd3cf5b200456b326192fe6d0a9c5f: Writing region info on filesystem at 1733286522379Initializing all the Stores at 1733286522380 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286522380Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286522380Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286522381 (+1 ms)Obtaining lock to block concurrent updates at 1733286522405 (+24 ms)Preparing flush snapshotting stores in 9bfd3cf5b200456b326192fe6d0a9c5f at 1733286522405Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733286522405Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. at 1733286522405Flushing 9bfd3cf5b200456b326192fe6d0a9c5f/a: creating writer at 1733286522405Flushing 9bfd3cf5b200456b326192fe6d0a9c5f/a: appending metadata at 1733286522420 (+15 ms)Flushing 9bfd3cf5b200456b326192fe6d0a9c5f/a: closing flushed file at 1733286522420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32723f3b: reopening flushed file at 1733286522458 (+38 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 9bfd3cf5b200456b326192fe6d0a9c5f in 62ms, sequenceid=5, compaction requested=false; wal=null at 1733286522467 (+9 ms)Cleaning up temporary data from old regions at 1733286522474 (+7 ms)Region opened successfully at 1733286522482 (+8 ms) 2024-12-04T04:28:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bfd3cf5b200456b326192fe6d0a9c5f, disabling compactions & flushes 2024-12-04T04:28:42,493 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. after waiting 0 ms 2024-12-04T04:28:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:42,498 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733286518052.9bfd3cf5b200456b326192fe6d0a9c5f. 2024-12-04T04:28:42,498 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bfd3cf5b200456b326192fe6d0a9c5f: Waiting for close lock at 1733286522493Disabling compacts and flushes for region at 1733286522493Disabling writes for close at 1733286522493Writing region close event to WAL at 1733286522498 (+5 ms)Closed at 1733286522498 2024-12-04T04:28:42,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741883_1061 (size=93) 2024-12-04T04:28:42,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741883_1061 (size=93) 2024-12-04T04:28:42,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741883_1061 (size=93) 2024-12-04T04:28:42,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:42,517 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733286522361) 2024-12-04T04:28:42,533 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=405 (was 397) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:35021 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_788959477_22 at /127.0.0.1:58968 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35021 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_788959477_22 at /127.0.0.1:33754 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=751 (was 791), ProcessCount=11 (was 11), AvailableMemoryMB=5294 (was 5314) 2024-12-04T04:28:42,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=405, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=751, ProcessCount=11, AvailableMemoryMB=5294 2024-12-04T04:28:42,559 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:42,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T04:28:42,568 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:42,570 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6927651d 2024-12-04T04:28:42,571 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T04:28:42,573 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T04:28:42,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T04:28:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T04:28:42,586 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T04:28:42,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-04T04:28:42,588 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,590 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T04:28:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T04:28:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741885_1063 (size=694) 2024-12-04T04:28:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741885_1063 (size=694) 2024-12-04T04:28:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741885_1063 (size=694) 2024-12-04T04:28:42,609 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205 2024-12-04T04:28:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741886_1064 (size=77) 2024-12-04T04:28:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741886_1064 (size=77) 2024-12-04T04:28:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741886_1064 (size=77) 2024-12-04T04:28:42,622 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:42,623 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:42,623 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:42,623 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:42,623 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:42,623 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:42,623 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:42,623 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286522622Disabling compacts and flushes for region at 1733286522622Disabling writes for close at 1733286522623 (+1 ms)Writing region close event to WAL at 1733286522623Closed at 1733286522623 2024-12-04T04:28:42,625 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T04:28:42,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733286522625"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733286522625"}]},"ts":"1733286522625"} 2024-12-04T04:28:42,634 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T04:28:42,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T04:28:42,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733286522636"}]},"ts":"1733286522636"} 2024-12-04T04:28:42,643 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-04T04:28:42,643 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {67abd3a0ee76=0} racks are {/default-rack=0} 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T04:28:42,645 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T04:28:42,645 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T04:28:42,645 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T04:28:42,645 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T04:28:42,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN}] 2024-12-04T04:28:42,649 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN 2024-12-04T04:28:42,650 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN; state=OFFLINE, location=67abd3a0ee76,41303,1733286503326; forceNewPlan=false, retain=false 2024-12-04T04:28:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T04:28:42,802 INFO [67abd3a0ee76:33125 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T04:28:42,803 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPENING, regionLocation=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:42,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN because future has completed 2024-12-04T04:28:42,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326}] 2024-12-04T04:28:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T04:28:42,964 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T04:28:42,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T04:28:42,973 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:42,973 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:42,974 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,974 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:42,974 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,974 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,976 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,978 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:42,978 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,979 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:42,979 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,981 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:42,981 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:42,982 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:42,982 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,984 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,984 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,985 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,985 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,986 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T04:28:42,988 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,991 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:42,992 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3c8da809e64ee3e582bbcab19c4b44fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65926304, jitterRate=-0.017621517181396484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T04:28:42,992 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:42,993 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3c8da809e64ee3e582bbcab19c4b44fb: Running coprocessor pre-open hook at 1733286522974Writing region info on filesystem at 1733286522974Initializing all the Stores at 1733286522975 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286522975Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286522976 (+1 ms)Cleaning up temporary data from old regions at 1733286522985 (+9 ms)Running coprocessor post-open hooks at 1733286522992 (+7 ms)Region opened successfully at 1733286522993 (+1 ms) 2024-12-04T04:28:42,995 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., pid=6, masterSystemTime=1733286522964 2024-12-04T04:28:42,999 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,000 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPEN, openSeqNum=2, regionLocation=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:43,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 because future has completed 2024-12-04T04:28:43,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T04:28:43,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 in 203 msec 2024-12-04T04:28:43,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T04:28:43,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN in 373 msec 2024-12-04T04:28:43,027 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T04:28:43,027 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733286523027"}]},"ts":"1733286523027"} 2024-12-04T04:28:43,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-04T04:28:43,034 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T04:28:43,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 455 msec 2024-12-04T04:28:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T04:28:43,227 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-04T04:28:43,227 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-04T04:28:43,229 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T04:28:43,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-04T04:28:43,238 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T04:28:43,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-04T04:28:43,257 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=2] 2024-12-04T04:28:43,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T04:28:43,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T04:28:43,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3c8da809e64ee3e582bbcab19c4b44fb, source=67abd3a0ee76,41303,1733286503326, destination=67abd3a0ee76,40991,1733286503128, warming up region on 67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T04:28:43,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3c8da809e64ee3e582bbcab19c4b44fb, source=67abd3a0ee76,41303,1733286503326, destination=67abd3a0ee76,40991,1733286503128, running balancer 2024-12-04T04:28:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE 2024-12-04T04:28:43,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T04:28:43,301 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE 2024-12-04T04:28:43,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=CLOSING, regionLocation=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:43,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE because future has completed 2024-12-04T04:28:43,308 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T04:28:43,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326}] 2024-12-04T04:28:43,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:43,332 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:43,336 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:43,336 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:43,339 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:43,339 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:43,341 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:43,341 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:43,343 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:43,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286523343Disabling compacts and flushes for region at 1733286523343Disabling writes for close at 1733286523344 (+1 ms)Writing region close event to WAL at 1733286523344Closed at 1733286523344 2024-12-04T04:28:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T04:28:43,480 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:43,481 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T04:28:43,481 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:43,482 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,482 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,482 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:43,482 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:43,482 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 3c8da809e64ee3e582bbcab19c4b44fb 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-04T04:28:43,516 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/af8fc44bf1b64329a7287a0b9335a911 is 35, key is r1/cf1:q/1733286523274/Put/seqid=0 2024-12-04T04:28:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741887_1065 (size=4783) 2024-12-04T04:28:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741887_1065 (size=4783) 2024-12-04T04:28:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741887_1065 (size=4783) 2024-12-04T04:28:43,982 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/af8fc44bf1b64329a7287a0b9335a911 2024-12-04T04:28:43,989 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/af8fc44bf1b64329a7287a0b9335a911 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911 2024-12-04T04:28:43,996 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911, entries=1, sequenceid=5, filesize=4.7 K 2024-12-04T04:28:43,997 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3c8da809e64ee3e582bbcab19c4b44fb in 515ms, sequenceid=5, compaction requested=false 2024-12-04T04:28:43,997 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T04:28:44,002 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T04:28:44,004 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:44,005 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286523481Running coprocessor pre-close hooks at 1733286523481Disabling compacts and flushes for region at 1733286523481Disabling writes for close at 1733286523482 (+1 ms)Obtaining lock to block concurrent updates at 1733286523482Preparing flush snapshotting stores in 3c8da809e64ee3e582bbcab19c4b44fb at 1733286523482Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733286523483 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. at 1733286523490 (+7 ms)Flushing 3c8da809e64ee3e582bbcab19c4b44fb/cf1: creating writer at 1733286523490Flushing 3c8da809e64ee3e582bbcab19c4b44fb/cf1: appending metadata at 1733286523515 (+25 ms)Flushing 3c8da809e64ee3e582bbcab19c4b44fb/cf1: closing flushed file at 1733286523516 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71a97d15: reopening flushed file at 1733286523988 (+472 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3c8da809e64ee3e582bbcab19c4b44fb in 515ms, sequenceid=5, compaction requested=false at 1733286523997 (+9 ms)Writing region close event to WAL at 1733286523998 (+1 ms)Running coprocessor post-close hooks at 1733286524003 (+5 ms)Closed at 1733286524004 (+1 ms) 2024-12-04T04:28:44,005 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3c8da809e64ee3e582bbcab19c4b44fb move to 67abd3a0ee76,40991,1733286503128 record at close sequenceid=5 2024-12-04T04:28:44,008 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=CLOSED 2024-12-04T04:28:44,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 because future has completed 2024-12-04T04:28:44,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T04:28:44,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 in 704 msec 2024-12-04T04:28:44,018 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE; state=CLOSED, location=67abd3a0ee76,40991,1733286503128; forceNewPlan=false, retain=false 2024-12-04T04:28:44,169 INFO [67abd3a0ee76:33125 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T04:28:44,170 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPENING, regionLocation=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:44,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE because future has completed 2024-12-04T04:28:44,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128}] 2024-12-04T04:28:44,334 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:44,335 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:44,335 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,335 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:44,336 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,336 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,338 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,339 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:44,339 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:44,350 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911 2024-12-04T04:28:44,350 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:44,350 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,352 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:44,352 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:44,352 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:44,353 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,353 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,354 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,355 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,355 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,356 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T04:28:44,358 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,359 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3c8da809e64ee3e582bbcab19c4b44fb; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59967058, jitterRate=-0.10642120242118835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T04:28:44,359 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:44,360 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3c8da809e64ee3e582bbcab19c4b44fb: Running coprocessor pre-open hook at 1733286524336Writing region info on filesystem at 1733286524336Initializing all the Stores at 1733286524337 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286524337Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286524338 (+1 ms)Cleaning up temporary data from old regions at 1733286524355 (+17 ms)Running coprocessor post-open hooks at 1733286524360 (+5 ms)Region opened successfully at 1733286524360 2024-12-04T04:28:44,364 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., pid=9, masterSystemTime=1733286524330 2024-12-04T04:28:44,367 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:44,368 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:44,368 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPEN, openSeqNum=9, regionLocation=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:44,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128 because future has completed 2024-12-04T04:28:44,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T04:28:44,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128 in 197 msec 2024-12-04T04:28:44,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE in 1.0800 sec 2024-12-04T04:28:44,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T04:28:44,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T04:28:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:59924 deadline: 1733286584421, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=40991 startCode=1733286503128. As of locationSeqNum=5. 2024-12-04T04:28:44,446 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=40991 startCode=1733286503128. As of locationSeqNum=5. 2024-12-04T04:28:44,447 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=40991 startCode=1733286503128. As of locationSeqNum=5. 2024-12-04T04:28:44,447 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,40991,1733286503128, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=40991 startCode=1733286503128. As of locationSeqNum=5. 2024-12-04T04:28:44,556 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T04:28:44,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T04:28:44,584 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3c8da809e64ee3e582bbcab19c4b44fb 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-04T04:28:44,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/22635a21bff24dc99be49425cf8c73cf is 29, key is r1/cf1:/1733286524560/DeleteFamily/seqid=0 2024-12-04T04:28:44,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741888_1066 (size=4906) 2024-12-04T04:28:44,629 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741888_1066 (size=4906) 2024-12-04T04:28:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741888_1066 (size=4906) 2024-12-04T04:28:44,638 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:44,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 is 29, key is r1/cf2:/1733286524560/DeleteFamily/seqid=0 2024-12-04T04:28:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741889_1067 (size=4906) 2024-12-04T04:28:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741889_1067 (size=4906) 2024-12-04T04:28:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741889_1067 (size=4906) 2024-12-04T04:28:45,111 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,119 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/22635a21bff24dc99be49425cf8c73cf as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:45,128 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:45,128 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T04:28:45,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,138 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T04:28:45,140 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 3c8da809e64ee3e582bbcab19c4b44fb in 556ms, sequenceid=12, compaction requested=false 2024-12-04T04:28:45,140 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3c8da809e64ee3e582bbcab19c4b44fb: 2024-12-04T04:28:45,143 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T04:28:45,145 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3c8da809e64ee3e582bbcab19c4b44fb/cf1 is initiating major compaction (all files) 2024-12-04T04:28:45,145 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T04:28:45,145 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:45,146 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3c8da809e64ee3e582bbcab19c4b44fb/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,147 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911, hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf] into tmpdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp, totalSize=9.5 K 2024-12-04T04:28:45,148 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting af8fc44bf1b64329a7287a0b9335a911, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733286523274 2024-12-04T04:28:45,150 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 22635a21bff24dc99be49425cf8c73cf, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T04:28:45,168 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3c8da809e64ee3e582bbcab19c4b44fb#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T04:28:45,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741890_1068 (size=4626) 2024-12-04T04:28:45,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741890_1068 (size=4626) 2024-12-04T04:28:45,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741890_1068 (size=4626) 2024-12-04T04:28:45,194 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf1/92dc5d6325404d5d9149acc2d621aed4 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/92dc5d6325404d5d9149acc2d621aed4 2024-12-04T04:28:45,214 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 3c8da809e64ee3e582bbcab19c4b44fb/cf1 of 3c8da809e64ee3e582bbcab19c4b44fb into 92dc5d6325404d5d9149acc2d621aed4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T04:28:45,215 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3c8da809e64ee3e582bbcab19c4b44fb: 2024-12-04T04:28:45,215 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T04:28:45,215 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3c8da809e64ee3e582bbcab19c4b44fb/cf2 is initiating major compaction (all files) 2024-12-04T04:28:45,215 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T04:28:45,215 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T04:28:45,215 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3c8da809e64ee3e582bbcab19c4b44fb/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,215 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7] into tmpdir=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp, totalSize=4.8 K 2024-12-04T04:28:45,216 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ccbfecdda2ef4fd6b57a6099f64edbd7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T04:28:45,233 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3c8da809e64ee3e582bbcab19c4b44fb#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T04:28:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741891_1069 (size=4592) 2024-12-04T04:28:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741891_1069 (size=4592) 2024-12-04T04:28:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741891_1069 (size=4592) 2024-12-04T04:28:45,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/.tmp/cf2/2b0a78bdadc04d94a7dd61979ac3824f as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/2b0a78bdadc04d94a7dd61979ac3824f 2024-12-04T04:28:45,292 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 3c8da809e64ee3e582bbcab19c4b44fb/cf2 of 3c8da809e64ee3e582bbcab19c4b44fb into 2b0a78bdadc04d94a7dd61979ac3824f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T04:28:45,292 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3c8da809e64ee3e582bbcab19c4b44fb: 2024-12-04T04:28:45,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3c8da809e64ee3e582bbcab19c4b44fb, source=67abd3a0ee76,40991,1733286503128, destination=67abd3a0ee76,41303,1733286503326, warming up region on 67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:45,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3c8da809e64ee3e582bbcab19c4b44fb, source=67abd3a0ee76,40991,1733286503128, destination=67abd3a0ee76,41303,1733286503326, running balancer 2024-12-04T04:28:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE 2024-12-04T04:28:45,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:45,302 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE 2024-12-04T04:28:45,303 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,304 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=CLOSING, regionLocation=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:45,304 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:45,304 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:45,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE because future has completed 2024-12-04T04:28:45,307 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T04:28:45,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128}] 2024-12-04T04:28:45,313 INFO [StoreFileOpener-3c8da809e64ee3e582bbcab19c4b44fb-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:45,313 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:45,321 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/92dc5d6325404d5d9149acc2d621aed4 2024-12-04T04:28:45,328 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911 2024-12-04T04:28:45,328 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:45,329 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,330 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:45,330 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:45,337 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/2b0a78bdadc04d94a7dd61979ac3824f 2024-12-04T04:28:45,344 INFO [StoreFileOpener-3c8da809e64ee3e582bbcab19c4b44fb-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,344 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,345 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:45,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41303 {}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286525345Disabling compacts and flushes for region at 1733286525345Disabling writes for close at 1733286525345Writing region close event to WAL at 1733286525346 (+1 ms)Closed at 1733286525346 2024-12-04T04:28:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-04T04:28:45,463 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,463 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T04:28:45,463 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:45,463 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,463 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,463 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:45,463 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,463 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911, hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf] to archive 2024-12-04T04:28:45,466 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T04:28:45,469 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911 to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/af8fc44bf1b64329a7287a0b9335a911 2024-12-04T04:28:45,471 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/22635a21bff24dc99be49425cf8c73cf 2024-12-04T04:28:45,490 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7] to archive 2024-12-04T04:28:45,491 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T04:28:45,493 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/ccbfecdda2ef4fd6b57a6099f64edbd7 2024-12-04T04:28:45,500 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-04T04:28:45,502 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,502 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286525463Running coprocessor pre-close hooks at 1733286525463Disabling compacts and flushes for region at 1733286525463Disabling writes for close at 1733286525463Writing region close event to WAL at 1733286525496 (+33 ms)Running coprocessor post-close hooks at 1733286525501 (+5 ms)Closed at 1733286525501 2024-12-04T04:28:45,502 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 3c8da809e64ee3e582bbcab19c4b44fb move to 67abd3a0ee76,41303,1733286503326 record at close sequenceid=12 2024-12-04T04:28:45,509 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,509 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=CLOSED 2024-12-04T04:28:45,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128 because future has completed 2024-12-04T04:28:45,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T04:28:45,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,40991,1733286503128 in 208 msec 2024-12-04T04:28:45,523 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE; state=CLOSED, location=67abd3a0ee76,41303,1733286503326; forceNewPlan=false, retain=false 2024-12-04T04:28:45,674 INFO [67abd3a0ee76:33125 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T04:28:45,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPENING, regionLocation=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:45,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE because future has completed 2024-12-04T04:28:45,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326}] 2024-12-04T04:28:45,836 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,836 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:45,836 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,836 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:45,836 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,836 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,838 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,840 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:45,840 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:45,849 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/92dc5d6325404d5d9149acc2d621aed4 2024-12-04T04:28:45,849 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:45,849 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,850 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:45,850 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:45,859 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/2b0a78bdadc04d94a7dd61979ac3824f 2024-12-04T04:28:45,860 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:45,860 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,861 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,863 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,864 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,864 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,865 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T04:28:45,867 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,869 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3c8da809e64ee3e582bbcab19c4b44fb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71950492, jitterRate=0.07214587926864624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T04:28:45,869 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,870 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3c8da809e64ee3e582bbcab19c4b44fb: Running coprocessor pre-open hook at 1733286525837Writing region info on filesystem at 1733286525837Initializing all the Stores at 1733286525838 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286525838Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286525838Cleaning up temporary data from old regions at 1733286525864 (+26 ms)Running coprocessor post-open hooks at 1733286525869 (+5 ms)Region opened successfully at 1733286525870 (+1 ms) 2024-12-04T04:28:45,871 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., pid=12, masterSystemTime=1733286525831 2024-12-04T04:28:45,877 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,877 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,881 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPEN, openSeqNum=18, regionLocation=67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:45,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 because future has completed 2024-12-04T04:28:45,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-04T04:28:45,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,41303,1733286503326 in 210 msec 2024-12-04T04:28:45,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, REOPEN/MOVE in 594 msec 2024-12-04T04:28:45,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T04:28:45,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T04:28:45,907 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 67abd3a0ee76,41303,1733286503326: testing ***** 2024-12-04T04:28:45,907 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T04:28:45,910 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T04:28:45,911 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T04:28:45,917 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T04:28:45,921 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T04:28:45,942 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 287020272 }, "NonHeapMemoryUsage": { "committed": 171769856, "init": 7667712, "max": -1, "used": 169071288 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "67abd3a0ee76", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 49, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 43, "ProcessCallTime_98th_percentile": 49, "ProcessCallTime_99th_percentile": 49, "ProcessCallTime_99.9th_percentile": 49, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 49, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 43, "TotalCallTime_98th_percentile": 49, "TotalCallTime_99th_percentile": 49, "TotalCallTime_99.9th_percentile": 49, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 158, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5744 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "67abd3a0ee76", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:50801", "tag.serverName": "67abd3a0ee76,40991,1733286503128", "tag.clusterId": "436670bb-39cf-467e-9f7a-82d611d5e88b", "tag.Context": "regionserver", "tag.Hostname": "67abd3a0ee76", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733286503128, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 66, "MajorCompactionTime_max": 76, "MajorCompactionTime_mean": 71, "MajorCompactionTime_25th_percentile": 66, "MajorCompactionTime_median": 67, "MajorCompactionTime_75th_percentile": 76, "MajorCompactionTime_90th_percentile": 76, "MajorCompactionTime_95th_percentile": 76, "MajorCompactionTime_98th_percentile": 76, "MajorCompactionTime_99th_percentile": 76, "MajorCompactionTime_99.9th_percentile": 76, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 1, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 13, "Delete_max": 13, "Delete_mean": 13, "Delete_25th_percentile": 13, "Delete_median": 13, "Delete_75th_percentile": 13, "Delete_90th_percentile": 13, "Delete_95th_percentile": 13, "Delete_98th_percentile": 13, "Delete_99th_percentile": 13, "Delete_99.9th_percentile": 13, "Delete_TimeRangeCount_10-30": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 66, "CompactionTime_max": 76, "CompactionTime_mean": 71, "CompactionTime_25th_percentile": 66, "CompactionTime_median": 67, "CompactionTime_75th_percentile": 76, "CompactionTime_90th_percentile": 76, "CompactionTime_95th_percentile": 76, "CompactionTime_98th_percentile": 76, "CompactionTime_99th_percentile": 76, "CompactionTime_99.9th_percentile": 76, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 556, "FlushTime_max": 556, "FlushTime_mean": 556, "FlushTime_25th_percentile": 556, "FlushTime_median": 556, "FlushTime_75th_percentile": 556, "FlushTime_90th_percentile": 556, "FlushTime_95th_percentile": 556, "FlushTime_98th_percentile": 556, "FlushTime_99th_percentile": 556, "FlushTime_99.9th_percentile": 556, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-04T04:28:45,946 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33125 {}] master.MasterRpcServices(700): 67abd3a0ee76,41303,1733286503326 reported a fatal error: ***** ABORTING region server 67abd3a0ee76,41303,1733286503326: testing ***** 2024-12-04T04:28:45,949 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '67abd3a0ee76,41303,1733286503326' ***** 2024-12-04T04:28:45,949 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-04T04:28:45,950 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T04:28:45,950 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T04:28:45,950 INFO [RS:1;67abd3a0ee76:41303 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-04T04:28:45,950 INFO [RS:1;67abd3a0ee76:41303 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-04T04:28:45,950 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,951 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(956): aborting server 67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:45,951 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T04:28:45,951 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:28:45,951 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,951 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,951 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:28:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40991 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:54722 deadline: 1733286585951, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=41303 startCode=1733286503326. As of locationSeqNum=12. 2024-12-04T04:28:45,951 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,952 INFO [RS:1;67abd3a0ee76:41303 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;67abd3a0ee76:41303. 2024-12-04T04:28:45,952 DEBUG [RS:1;67abd3a0ee76:41303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T04:28:45,952 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,40991,1733286503128, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,40991,1733286503128, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=41303 startCode=1733286503326. As of locationSeqNum=12. 2024-12-04T04:28:45,952 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,40991,1733286503128, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=41303 startCode=1733286503326. As of locationSeqNum=12. 2024-12-04T04:28:45,952 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,40991,1733286503128, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=67abd3a0ee76 port=41303 startCode=1733286503326. As of locationSeqNum=12. 2024-12-04T04:28:45,953 DEBUG [RS:1;67abd3a0ee76:41303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:28:45,953 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T04:28:45,953 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1325): Online Regions={3c8da809e64ee3e582bbcab19c4b44fb=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.} 2024-12-04T04:28:45,954 DEBUG [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1351): Waiting on 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:45,957 INFO [regionserver/67abd3a0ee76:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:28:45,963 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:45,963 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286525951Running coprocessor pre-close hooks at 1733286525951Disabling compacts and flushes for region at 1733286525951Disabling writes for close at 1733286525951Writing region close event to WAL at 1733286525962 (+11 ms)Running coprocessor post-close hooks at 1733286525962Closed at 1733286525963 (+1 ms) 2024-12-04T04:28:45,963 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:46,036 INFO [regionserver/67abd3a0ee76:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T04:28:46,036 INFO [regionserver/67abd3a0ee76:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T04:28:46,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 67abd3a0ee76,41303,1733286503326 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:46,067 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 67abd3a0ee76,41303,1733286503326 aborting 2024-12-04T04:28:46,068 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 67abd3a0ee76,41303,1733286503326 aborting 2024-12-04T04:28:46,068 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=12 from cache 2024-12-04T04:28:46,154 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(976): stopping server 67abd3a0ee76,41303,1733286503326; all regions closed. 2024-12-04T04:28:46,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741835_1011 (size=1677) 2024-12-04T04:28:46,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741835_1011 (size=1677) 2024-12-04T04:28:46,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741835_1011 (size=1677) 2024-12-04T04:28:46,159 DEBUG [RS:1;67abd3a0ee76:41303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:28:46,159 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:28:46,159 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T04:28:46,160 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.ChoreService(370): Chore service for: regionserver/67abd3a0ee76:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T04:28:46,160 INFO [regionserver/67abd3a0ee76:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T04:28:46,160 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T04:28:46,160 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T04:28:46,160 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T04:28:46,160 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T04:28:46,161 INFO [RS:1;67abd3a0ee76:41303 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41303 2024-12-04T04:28:46,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/67abd3a0ee76,41303,1733286503326 2024-12-04T04:28:46,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T04:28:46,206 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T04:28:46,207 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [67abd3a0ee76,41303,1733286503326] 2024-12-04T04:28:46,280 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18] 2024-12-04T04:28:46,282 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 67abd3a0ee76:41303 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 67abd3a0ee76/172.17.0.3:41303 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:46,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18, error=java.net.ConnectException: Call to address=67abd3a0ee76:41303 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 67abd3a0ee76/172.17.0.3:41303 2024-12-04T04:28:46,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-04T04:28:46,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 from cache 2024-12-04T04:28:46,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 67abd3a0ee76:41303 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 67abd3a0ee76/172.17.0.3:41303 2024-12-04T04:28:46,348 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/67abd3a0ee76,41303,1733286503326 already deleted, retry=false 2024-12-04T04:28:46,348 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 67abd3a0ee76,41303,1733286503326 on 67abd3a0ee76,33125,1733286501754 2024-12-04T04:28:46,354 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 67abd3a0ee76,41303,1733286503326, splitWal=true, meta=false 2024-12-04T04:28:46,356 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 67abd3a0ee76,41303,1733286503326 (carryingMeta=false) 67abd3a0ee76,41303,1733286503326/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@325f118[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T04:28:46,357 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 67abd3a0ee76,41303,1733286503326, splitWal=true, meta=false 2024-12-04T04:28:46,360 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 67abd3a0ee76,41303,1733286503326 had 1 regions 2024-12-04T04:28:46,362 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 67abd3a0ee76,41303,1733286503326, splitWal=true, meta=false, isMeta: false 2024-12-04T04:28:46,364 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting 2024-12-04T04:28:46,365 INFO [PEWorker-2 {}] master.SplitWALManager(105): 67abd3a0ee76,41303,1733286503326 WAL count=1, meta=false 2024-12-04T04:28:46,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 67abd3a0ee76%2C41303%2C1733286503326.1733286506183}] 2024-12-04T04:28:46,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:28:46,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41303-0x1019e9915430002, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:28:46,370 INFO [RS:1;67abd3a0ee76:41303 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T04:28:46,370 INFO [RS:1;67abd3a0ee76:41303 {}] regionserver.HRegionServer(1031): Exiting; stopping=67abd3a0ee76,41303,1733286503326; zookeeper connection closed. 2024-12-04T04:28:46,371 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4592d633 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4592d633 2024-12-04T04:28:46,375 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:46,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 67abd3a0ee76%2C41303%2C1733286503326.1733286506183, worker=67abd3a0ee76,40991,1733286503128}] 2024-12-04T04:28:46,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40991 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-04T04:28:46,552 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, size=1.6 K (1677bytes) 2024-12-04T04:28:46,552 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 2024-12-04T04:28:46,553 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 after 1ms 2024-12-04T04:28:46,555 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:46,555 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 took 3ms 2024-12-04T04:28:46,562 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 3c8da809e64ee3e582bbcab19c4b44fb: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-04T04:28:46,562 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 so closing down 2024-12-04T04:28:46,562 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:46,562 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:46,562 INFO [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-04T04:28:46,563 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, journal: Splitting hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, size=1.6 K (1677bytes) at 1733286526552Finishing writing output for hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 so closing down at 1733286526562 (+10 ms)3 split writer threads finished at 1733286526562Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733286526562 2024-12-04T04:28:46,563 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 2024-12-04T04:28:46,564 DEBUG [RS_LOG_REPLAY_OPS-regionserver/67abd3a0ee76:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T04:28:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33125 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-04T04:28:46,570 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting/67abd3a0ee76%2C41303%2C1733286503326.1733286506183 to hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs 2024-12-04T04:28:46,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T04:28:46,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 67abd3a0ee76%2C41303%2C1733286503326.1733286506183, worker=67abd3a0ee76,40991,1733286503128 in 193 msec 2024-12-04T04:28:46,574 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=67abd3a0ee76,40991,1733286503128 2024-12-04T04:28:46,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T04:28:46,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 67abd3a0ee76%2C41303%2C1733286503326.1733286506183, worker=67abd3a0ee76,40991,1733286503128 in 207 msec 2024-12-04T04:28:46,580 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting dir is empty, no logs to split. 2024-12-04T04:28:46,580 INFO [PEWorker-2 {}] master.SplitWALManager(105): 67abd3a0ee76,41303,1733286503326 WAL count=0, meta=false 2024-12-04T04:28:46,580 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 67abd3a0ee76,41303,1733286503326 WAL splitting is done? wals=0, meta=false 2024-12-04T04:28:46,582 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 67abd3a0ee76,41303,1733286503326 failed, ignore...File hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/WALs/67abd3a0ee76,41303,1733286503326-splitting does not exist. 2024-12-04T04:28:46,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN}] 2024-12-04T04:28:46,587 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN 2024-12-04T04:28:46,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18] 2024-12-04T04:28:46,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 67abd3a0ee76:41303 this server is in the failed servers list 2024-12-04T04:28:46,588 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T04:28:46,589 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=67abd3a0ee76:41303 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 67abd3a0ee76:41303 2024-12-04T04:28:46,589 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 67abd3a0ee76:41303 2024-12-04T04:28:46,589 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 from cache 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(204): Hosts are {67abd3a0ee76=0} racks are {/default-rack=0} 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T04:28:46,739 INFO [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T04:28:46,739 INFO [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T04:28:46,739 DEBUG [67abd3a0ee76:33125 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T04:28:46,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPENING, regionLocation=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:46,933 INFO [AsyncFSWAL-0-hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205-prefix:67abd3a0ee76,35953,1733286503448.meta {}] wal.AbstractFSWAL(1368): Slow sync cost: 191 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK], DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK], DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK]] 2024-12-04T04:28:46,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN because future has completed 2024-12-04T04:28:46,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,35953,1733286503448}] 2024-12-04T04:28:47,092 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:47,092 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c8da809e64ee3e582bbcab19c4b44fb, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:47,092 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,092 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:47,092 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,092 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,094 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,095 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf1 2024-12-04T04:28:47,095 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:47,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18] 2024-12-04T04:28:47,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 67abd3a0ee76:41303 this server is in the failed servers list 2024-12-04T04:28:47,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=67abd3a0ee76:41303 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 67abd3a0ee76:41303 2024-12-04T04:28:47,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 67abd3a0ee76:41303 2024-12-04T04:28:47,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,41303,1733286503326, seqNum=18 from cache 2024-12-04T04:28:47,106 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf1/92dc5d6325404d5d9149acc2d621aed4 2024-12-04T04:28:47,106 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:47,107 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,108 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c8da809e64ee3e582bbcab19c4b44fb columnFamilyName cf2 2024-12-04T04:28:47,108 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:47,119 DEBUG [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/cf2/2b0a78bdadc04d94a7dd61979ac3824f 2024-12-04T04:28:47,119 INFO [StoreOpener-3c8da809e64ee3e582bbcab19c4b44fb-1 {}] regionserver.HStore(327): Store=3c8da809e64ee3e582bbcab19c4b44fb/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:47,119 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,120 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,125 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,126 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,126 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,126 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T04:28:47,130 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,131 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 3c8da809e64ee3e582bbcab19c4b44fb; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71852625, jitterRate=0.07068754732608795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T04:28:47,132 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:28:47,133 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 3c8da809e64ee3e582bbcab19c4b44fb: Running coprocessor pre-open hook at 1733286527093Writing region info on filesystem at 1733286527093Initializing all the Stores at 1733286527093Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286527094 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286527094Cleaning up temporary data from old regions at 1733286527126 (+32 ms)Running coprocessor post-open hooks at 1733286527132 (+6 ms)Region opened successfully at 1733286527133 (+1 ms) 2024-12-04T04:28:47,134 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., pid=17, masterSystemTime=1733286527086 2024-12-04T04:28:47,136 DEBUG [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:47,137 INFO [RS_OPEN_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:28:47,138 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3c8da809e64ee3e582bbcab19c4b44fb, regionState=OPEN, openSeqNum=18, regionLocation=67abd3a0ee76,35953,1733286503448 2024-12-04T04:28:47,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,35953,1733286503448 because future has completed 2024-12-04T04:28:47,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T04:28:47,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c8da809e64ee3e582bbcab19c4b44fb, server=67abd3a0ee76,35953,1733286503448 in 208 msec 2024-12-04T04:28:47,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-04T04:28:47,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3c8da809e64ee3e582bbcab19c4b44fb, ASSIGN in 560 msec 2024-12-04T04:28:47,148 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 67abd3a0ee76,41303,1733286503326 after splitting done 2024-12-04T04:28:47,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 67abd3a0ee76,41303,1733286503326, splitWal=true, meta=false in 799 msec 2024-12-04T04:28:48,119 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., hostname=67abd3a0ee76,35953,1733286503448, seqNum=18] 2024-12-04T04:28:48,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=402 (was 405), OpenFileDescriptor=1042 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=739 (was 751), ProcessCount=11 (was 11), AvailableMemoryMB=4896 (was 5294) 2024-12-04T04:28:48,135 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-12-04T04:28:48,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=402, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=739, ProcessCount=11, AvailableMemoryMB=4895 2024-12-04T04:28:48,146 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-12-04T04:28:48,161 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:48,163 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:48,163 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:48,166 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54095756, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-54095756, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:48,178 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54095756/hregion-54095756.1733286528166, exclude list is [], retry=0 2024-12-04T04:28:48,181 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:48,181 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:48,182 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:48,185 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54095756/hregion-54095756.1733286528166 2024-12-04T04:28:48,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:48,186 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2ece6c84f3f7b1bc80b6310bd82f80b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:48,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741893_1071 (size=67) 2024-12-04T04:28:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741893_1071 (size=67) 2024-12-04T04:28:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741893_1071 (size=67) 2024-12-04T04:28:48,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:48,211 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,213 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName a 2024-12-04T04:28:48,213 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,214 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,214 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,216 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName b 2024-12-04T04:28:48,217 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,217 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,218 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,219 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName c 2024-12-04T04:28:48,219 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,221 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,221 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,223 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,237 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:48,239 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,245 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:48,246 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2ece6c84f3f7b1bc80b6310bd82f80b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64380832, jitterRate=-0.04065084457397461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:48,247 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: Writing region info on filesystem at 1733286528209Initializing all the Stores at 1733286528211 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528211Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528211Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528211Cleaning up temporary data from old regions at 1733286528236 (+25 ms)Region opened successfully at 1733286528247 (+11 ms) 2024-12-04T04:28:48,247 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2ece6c84f3f7b1bc80b6310bd82f80b7, disabling compactions & flushes 2024-12-04T04:28:48,247 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,247 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,248 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. after waiting 0 ms 2024-12-04T04:28:48,248 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,252 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,253 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: Waiting for close lock at 1733286528247Disabling compacts and flushes for region at 1733286528247Disabling writes for close at 1733286528248 (+1 ms)Writing region close event to WAL at 1733286528252 (+4 ms)Closed at 1733286528252 2024-12-04T04:28:48,264 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-54095756/hregion-54095756.1733286528166 not finished, retry = 0 2024-12-04T04:28:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741892_1070 (size=93) 2024-12-04T04:28:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741892_1070 (size=93) 2024-12-04T04:28:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741892_1070 (size=93) 2024-12-04T04:28:48,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:48,368 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54095756:(num 1733286528166) 2024-12-04T04:28:48,368 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:48,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:48,386 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, exclude list is [], retry=0 2024-12-04T04:28:48,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:48,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:48,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:48,397 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 2024-12-04T04:28:48,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:48,397 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ece6c84f3f7b1bc80b6310bd82f80b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:48,397 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:48,397 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,397 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,399 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,400 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName a 2024-12-04T04:28:48,400 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,401 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,401 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,402 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName b 2024-12-04T04:28:48,402 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,403 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,403 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,404 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName c 2024-12-04T04:28:48,404 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:48,405 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:48,405 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,406 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,411 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,411 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,412 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:48,417 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:48,419 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2ece6c84f3f7b1bc80b6310bd82f80b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71263319, jitterRate=0.061906203627586365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:48,420 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: Writing region info on filesystem at 1733286528398Initializing all the Stores at 1733286528399 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528399Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528399Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286528399Cleaning up temporary data from old regions at 1733286528411 (+12 ms)Region opened successfully at 1733286528420 (+9 ms) 2024-12-04T04:28:48,488 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2ece6c84f3f7b1bc80b6310bd82f80b7 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T04:28:48,511 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T04:28:48,523 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/a/9c169b3e97e3408d9dc1e391dee4d54b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733286528420/Put/seqid=0 2024-12-04T04:28:48,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741895_1073 (size=5958) 2024-12-04T04:28:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741895_1073 (size=5958) 2024-12-04T04:28:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741895_1073 (size=5958) 2024-12-04T04:28:48,565 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/a/9c169b3e97e3408d9dc1e391dee4d54b 2024-12-04T04:28:48,609 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/0a6a1cfe825e44ec8d0bfaeb8da3abda is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733286528442/Put/seqid=0 2024-12-04T04:28:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741896_1074 (size=5958) 2024-12-04T04:28:48,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741896_1074 (size=5958) 2024-12-04T04:28:48,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741896_1074 (size=5958) 2024-12-04T04:28:48,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/0a6a1cfe825e44ec8d0bfaeb8da3abda 2024-12-04T04:28:48,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/c/0cd9408b8ec847da860384f98182bd60 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733286528468/Put/seqid=0 2024-12-04T04:28:48,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741897_1075 (size=5958) 2024-12-04T04:28:48,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741897_1075 (size=5958) 2024-12-04T04:28:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741897_1075 (size=5958) 2024-12-04T04:28:48,699 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/c/0cd9408b8ec847da860384f98182bd60 2024-12-04T04:28:48,712 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/a/9c169b3e97e3408d9dc1e391dee4d54b as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/a/9c169b3e97e3408d9dc1e391dee4d54b 2024-12-04T04:28:48,731 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/a/9c169b3e97e3408d9dc1e391dee4d54b, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T04:28:48,734 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/0a6a1cfe825e44ec8d0bfaeb8da3abda as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/0a6a1cfe825e44ec8d0bfaeb8da3abda 2024-12-04T04:28:48,765 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/0a6a1cfe825e44ec8d0bfaeb8da3abda, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T04:28:48,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/c/0cd9408b8ec847da860384f98182bd60 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/c/0cd9408b8ec847da860384f98182bd60 2024-12-04T04:28:48,796 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/c/0cd9408b8ec847da860384f98182bd60, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T04:28:48,802 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2ece6c84f3f7b1bc80b6310bd82f80b7 in 314ms, sequenceid=33, compaction requested=false 2024-12-04T04:28:48,802 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: 2024-12-04T04:28:48,803 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2ece6c84f3f7b1bc80b6310bd82f80b7, disabling compactions & flushes 2024-12-04T04:28:48,803 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,803 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,803 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. after waiting 0 ms 2024-12-04T04:28:48,803 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,816 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. 2024-12-04T04:28:48,816 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: Waiting for close lock at 1733286528803Disabling compacts and flushes for region at 1733286528803Disabling writes for close at 1733286528803Writing region close event to WAL at 1733286528816 (+13 ms)Closed at 1733286528816 2024-12-04T04:28:48,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741894_1072 (size=2873) 2024-12-04T04:28:48,828 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 not finished, retry = 0 2024-12-04T04:28:48,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741894_1072 (size=2873) 2024-12-04T04:28:48,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741894_1072 (size=2873) 2024-12-04T04:28:48,947 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/0a6a1cfe825e44ec8d0bfaeb8da3abda to hdfs://localhost:35021/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/0a6a1cfe825e44ec8d0bfaeb8da3abda 2024-12-04T04:28:48,989 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, size=2.8 K (2873bytes) 2024-12-04T04:28:48,989 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 2024-12-04T04:28:48,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 after 0ms 2024-12-04T04:28:48,999 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:48,999 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 took 10ms 2024-12-04T04:28:49,018 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 so closing down 2024-12-04T04:28:49,018 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:49,025 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733286528371.temp 2024-12-04T04:28:49,032 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp 2024-12-04T04:28:49,033 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:49,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741898_1076 (size=2312) 2024-12-04T04:28:49,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741898_1076 (size=2312) 2024-12-04T04:28:49,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741898_1076 (size=2312) 2024-12-04T04:28:49,053 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:49,055 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 2024-12-04T04:28:49,055 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-04T04:28:49,055 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, journal: Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, size=2.8 K (2873bytes) at 1733286528989Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 so closing down at 1733286529018 (+29 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp at 1733286529032 (+14 ms)3 split writer threads finished at 1733286529033 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733286529053 (+20 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000003-wal.1733286528371.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 at 1733286529055 (+2 ms)Processed 32 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733286529055 2024-12-04T04:28:49,060 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286528371 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286528371 2024-12-04T04:28:49,062 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 2024-12-04T04:28:49,062 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:49,065 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:49,087 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286529066, exclude list is [], retry=0 2024-12-04T04:28:49,091 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:49,092 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:49,092 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:49,106 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286529066 2024-12-04T04:28:49,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:28:49,108 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ece6c84f3f7b1bc80b6310bd82f80b7, NAME => 'testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:49,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:49,109 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,109 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,112 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,114 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName a 2024-12-04T04:28:49,114 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,123 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/a/9c169b3e97e3408d9dc1e391dee4d54b 2024-12-04T04:28:49,124 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,124 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,130 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName b 2024-12-04T04:28:49,130 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,131 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,131 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,132 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ece6c84f3f7b1bc80b6310bd82f80b7 columnFamilyName c 2024-12-04T04:28:49,132 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,148 DEBUG [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/c/0cd9408b8ec847da860384f98182bd60 2024-12-04T04:28:49,148 INFO [StoreOpener-2ece6c84f3f7b1bc80b6310bd82f80b7-1 {}] regionserver.HStore(327): Store=2ece6c84f3f7b1bc80b6310bd82f80b7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,149 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,155 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 2024-12-04T04:28:49,159 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:49,161 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 2024-12-04T04:28:49,161 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2ece6c84f3f7b1bc80b6310bd82f80b7 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T04:28:49,183 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/b38afea6520f42a58862fb6906b4b81a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733286528442/Put/seqid=0 2024-12-04T04:28:49,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741900_1078 (size=5958) 2024-12-04T04:28:49,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741900_1078 (size=5958) 2024-12-04T04:28:49,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741900_1078 (size=5958) 2024-12-04T04:28:49,628 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/b38afea6520f42a58862fb6906b4b81a 2024-12-04T04:28:49,639 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/.tmp/b/b38afea6520f42a58862fb6906b4b81a as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/b38afea6520f42a58862fb6906b4b81a 2024-12-04T04:28:49,665 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/b/b38afea6520f42a58862fb6906b4b81a, entries=10, sequenceid=32, filesize=5.8 K 2024-12-04T04:28:49,665 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2ece6c84f3f7b1bc80b6310bd82f80b7 in 504ms, sequenceid=32, compaction requested=false; wal=null 2024-12-04T04:28:49,667 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/0000000000000000032 2024-12-04T04:28:49,669 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,669 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,672 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:49,675 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2ece6c84f3f7b1bc80b6310bd82f80b7 2024-12-04T04:28:49,679 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/2ece6c84f3f7b1bc80b6310bd82f80b7/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-04T04:28:49,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2ece6c84f3f7b1bc80b6310bd82f80b7; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60098184, jitterRate=-0.10446727275848389}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:49,682 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2ece6c84f3f7b1bc80b6310bd82f80b7: Writing region info on filesystem at 1733286529109Initializing all the Stores at 1733286529110 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529110Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529112 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529112Obtaining lock to block concurrent updates at 1733286529161 (+49 ms)Preparing flush snapshotting stores in 2ece6c84f3f7b1bc80b6310bd82f80b7 at 1733286529161Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733286529161Flushing stores of testReplayEditsWrittenViaHRegion,,1733286528162.2ece6c84f3f7b1bc80b6310bd82f80b7. at 1733286529161Flushing 2ece6c84f3f7b1bc80b6310bd82f80b7/b: creating writer at 1733286529161Flushing 2ece6c84f3f7b1bc80b6310bd82f80b7/b: appending metadata at 1733286529181 (+20 ms)Flushing 2ece6c84f3f7b1bc80b6310bd82f80b7/b: closing flushed file at 1733286529182 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b088d56: reopening flushed file at 1733286529637 (+455 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2ece6c84f3f7b1bc80b6310bd82f80b7 in 504ms, sequenceid=32, compaction requested=false; wal=null at 1733286529665 (+28 ms)Cleaning up temporary data from old regions at 1733286529670 (+5 ms)Region opened successfully at 1733286529682 (+12 ms) 2024-12-04T04:28:49,729 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=409 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:59062 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:33956 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:33754 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:59104 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:34854 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:34962 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1112 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=739 (was 739), ProcessCount=11 (was 11), AvailableMemoryMB=4677 (was 4895) 2024-12-04T04:28:49,729 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1112 is superior to 1024 2024-12-04T04:28:49,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=409, OpenFileDescriptor=1112, MaxFileDescriptor=1048576, SystemLoadAverage=739, ProcessCount=11, AvailableMemoryMB=4675 2024-12-04T04:28:49,745 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1112 is superior to 1024 2024-12-04T04:28:49,765 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:49,768 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:28:49,769 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:28:49,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08529771, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-08529771, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:49,795 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08529771/hregion-08529771.1733286529774, exclude list is [], retry=0 2024-12-04T04:28:49,801 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:49,801 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:49,805 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:49,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08529771/hregion-08529771.1733286529774 2024-12-04T04:28:49,826 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:28:49,826 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4a9519a3977d74ff845f1c0b4ca2cb8f, NAME => 'testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:28:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741902_1080 (size=68) 2024-12-04T04:28:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741902_1080 (size=68) 2024-12-04T04:28:49,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741902_1080 (size=68) 2024-12-04T04:28:49,854 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:49,864 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,866 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName a 2024-12-04T04:28:49,866 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,868 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,868 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,869 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName b 2024-12-04T04:28:49,870 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,870 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,870 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,872 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName c 2024-12-04T04:28:49,872 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:49,873 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:49,873 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,875 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,876 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,876 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,877 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:49,879 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:49,884 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:28:49,886 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a9519a3977d74ff845f1c0b4ca2cb8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73315549, jitterRate=0.09248681366443634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:49,887 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: Writing region info on filesystem at 1733286529854Initializing all the Stores at 1733286529856 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529856Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529860 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286529860Cleaning up temporary data from old regions at 1733286529876 (+16 ms)Region opened successfully at 1733286529887 (+11 ms) 2024-12-04T04:28:49,887 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4a9519a3977d74ff845f1c0b4ca2cb8f, disabling compactions & flushes 2024-12-04T04:28:49,887 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:49,887 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:49,887 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. after waiting 0 ms 2024-12-04T04:28:49,888 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:49,888 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:49,888 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: Waiting for close lock at 1733286529887Disabling compacts and flushes for region at 1733286529887Disabling writes for close at 1733286529888 (+1 ms)Writing region close event to WAL at 1733286529888Closed at 1733286529888 2024-12-04T04:28:49,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741901_1079 (size=93) 2024-12-04T04:28:49,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741901_1079 (size=93) 2024-12-04T04:28:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741901_1079 (size=93) 2024-12-04T04:28:49,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:28:49,914 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08529771:(num 1733286529774) 2024-12-04T04:28:49,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:49,918 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:49,939 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, exclude list is [], retry=0 2024-12-04T04:28:49,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:49,948 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:49,949 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:49,973 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 2024-12-04T04:28:49,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:28:50,071 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a9519a3977d74ff845f1c0b4ca2cb8f, NAME => 'testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:50,076 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,085 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,087 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName a 2024-12-04T04:28:50,087 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:50,089 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:50,089 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,092 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName b 2024-12-04T04:28:50,093 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:50,094 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:50,094 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,098 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName c 2024-12-04T04:28:50,098 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:50,100 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:50,100 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,101 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,110 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,110 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,112 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:28:50,117 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,123 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a9519a3977d74ff845f1c0b4ca2cb8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69988277, jitterRate=0.04290659725666046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:28:50,123 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:50,124 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: Running coprocessor pre-open hook at 1733286530077Writing region info on filesystem at 1733286530077Initializing all the Stores at 1733286530078 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286530078Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286530084 (+6 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286530084Cleaning up temporary data from old regions at 1733286530110 (+26 ms)Running coprocessor post-open hooks at 1733286530123 (+13 ms)Region opened successfully at 1733286530123 2024-12-04T04:28:50,196 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-04T04:28:50,201 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:50,456 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T04:28:50,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T04:28:51,202 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:52,203 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:52,363 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-04T04:28:52,364 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:52,365 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T04:28:52,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:52,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-04T04:28:52,367 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-04T04:28:53,204 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:54,204 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:55,205 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:55,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T04:28:56,205 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:57,206 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:58,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:59,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4a9519a3977d74ff845f1c0b4ca2cb8f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:28:59,209 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: 2024-12-04T04:28:59,209 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,228 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: 2024-12-04T04:28:59,229 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-04T04:28:59,229 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4a9519a3977d74ff845f1c0b4ca2cb8f, disabling compactions & flushes 2024-12-04T04:28:59,229 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,229 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,229 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. after waiting 0 ms 2024-12-04T04:28:59,229 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,229 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,230 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. 2024-12-04T04:28:59,230 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: Waiting for close lock at 1733286539229Running coprocessor pre-close hooks at 1733286539229Disabling compacts and flushes for region at 1733286539229Disabling writes for close at 1733286539229Writing region close event to WAL at 1733286539229Running coprocessor post-close hooks at 1733286539229Closed at 1733286539229 2024-12-04T04:28:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741903_1081 (size=2398) 2024-12-04T04:28:59,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741903_1081 (size=2398) 2024-12-04T04:28:59,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741903_1081 (size=2398) 2024-12-04T04:28:59,255 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, size=2.3 K (2398bytes) 2024-12-04T04:28:59,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 2024-12-04T04:28:59,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 after 0ms 2024-12-04T04:28:59,258 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:59,258 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 took 3ms 2024-12-04T04:28:59,262 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 so closing down 2024-12-04T04:28:59,262 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:28:59,265 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733286529919.temp 2024-12-04T04:28:59,268 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp 2024-12-04T04:28:59,269 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:28:59,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741904_1082 (size=1672) 2024-12-04T04:28:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741904_1082 (size=1672) 2024-12-04T04:28:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741904_1082 (size=1672) 2024-12-04T04:28:59,297 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-04T04:28:59,299 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 2024-12-04T04:28:59,299 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-04T04:28:59,299 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, journal: Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, size=2.3 K (2398bytes) at 1733286539255Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 so closing down at 1733286539262 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp at 1733286539268 (+6 ms)3 split writer threads finished at 1733286539269 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733286539297 (+28 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000004-wal.1733286529919.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 at 1733286539299 (+2 ms)Processed 23 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733286539299 2024-12-04T04:28:59,302 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286529919 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286529919 2024-12-04T04:28:59,303 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 2024-12-04T04:28:59,304 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:28:59,308 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:28:59,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286539308, exclude list is [], retry=0 2024-12-04T04:28:59,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:28:59,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:28:59,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:28:59,341 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286539308 2024-12-04T04:28:59,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:28:59,345 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a9519a3977d74ff845f1c0b4ca2cb8f, NAME => 'testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:28:59,345 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,345 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:28:59,346 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,346 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,349 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,350 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName a 2024-12-04T04:28:59,350 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:59,351 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:59,351 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,353 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName b 2024-12-04T04:28:59,353 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:59,354 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:59,354 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,355 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a9519a3977d74ff845f1c0b4ca2cb8f columnFamilyName c 2024-12-04T04:28:59,355 DEBUG [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:28:59,356 INFO [StoreOpener-4a9519a3977d74ff845f1c0b4ca2cb8f-1 {}] regionserver.HStore(327): Store=4a9519a3977d74ff845f1c0b4ca2cb8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:28:59,356 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,357 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,358 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:28:59,359 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 2024-12-04T04:28:59,361 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:28:59,363 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 2024-12-04T04:28:59,363 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-04T04:28:59,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/a/d28b7b906ff242bfa6e6abce6efccb03 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733286539214/Put/seqid=0 2024-12-04T04:28:59,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741906_1084 (size=5523) 2024-12-04T04:28:59,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741906_1084 (size=5523) 2024-12-04T04:28:59,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741906_1084 (size=5523) 2024-12-04T04:28:59,412 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/a/d28b7b906ff242bfa6e6abce6efccb03 2024-12-04T04:28:59,442 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/b/dbcf02a2a0f349cba975e2cd6c2ea52b is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733286539209/Put/seqid=0 2024-12-04T04:28:59,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741907_1085 (size=5524) 2024-12-04T04:28:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741907_1085 (size=5524) 2024-12-04T04:28:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741907_1085 (size=5524) 2024-12-04T04:28:59,852 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/b/dbcf02a2a0f349cba975e2cd6c2ea52b 2024-12-04T04:28:59,913 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/c/b5e78c40deae4e37be86ef50538df268 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733286539212/Put/seqid=0 2024-12-04T04:28:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741908_1086 (size=5457) 2024-12-04T04:28:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741908_1086 (size=5457) 2024-12-04T04:28:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741908_1086 (size=5457) 2024-12-04T04:29:00,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/c/b5e78c40deae4e37be86ef50538df268 2024-12-04T04:29:00,360 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/a/d28b7b906ff242bfa6e6abce6efccb03 as hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/a/d28b7b906ff242bfa6e6abce6efccb03 2024-12-04T04:29:00,370 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/a/d28b7b906ff242bfa6e6abce6efccb03, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T04:29:00,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/b/dbcf02a2a0f349cba975e2cd6c2ea52b as hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/b/dbcf02a2a0f349cba975e2cd6c2ea52b 2024-12-04T04:29:00,380 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/b/dbcf02a2a0f349cba975e2cd6c2ea52b, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T04:29:00,385 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/.tmp/c/b5e78c40deae4e37be86ef50538df268 as hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/c/b5e78c40deae4e37be86ef50538df268 2024-12-04T04:29:00,394 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/c/b5e78c40deae4e37be86ef50538df268, entries=6, sequenceid=26, filesize=5.3 K 2024-12-04T04:29:00,394 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4a9519a3977d74ff845f1c0b4ca2cb8f in 1031ms, sequenceid=26, compaction requested=false; wal=null 2024-12-04T04:29:00,396 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/0000000000000000026 2024-12-04T04:29:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:29:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:29:00,403 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:00,409 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:29:00,412 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsAfterAbortingFlush/4a9519a3977d74ff845f1c0b4ca2cb8f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-04T04:29:00,415 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a9519a3977d74ff845f1c0b4ca2cb8f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66809790, jitterRate=-0.004456549882888794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:00,415 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a9519a3977d74ff845f1c0b4ca2cb8f 2024-12-04T04:29:00,416 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a9519a3977d74ff845f1c0b4ca2cb8f: Running coprocessor pre-open hook at 1733286539346Writing region info on filesystem at 1733286539346Initializing all the Stores at 1733286539348 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286539348Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286539348Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286539348Obtaining lock to block concurrent updates at 1733286539363 (+15 ms)Preparing flush snapshotting stores in 4a9519a3977d74ff845f1c0b4ca2cb8f at 1733286539363Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733286539363Flushing stores of testReplayEditsAfterAbortingFlush,,1733286529766.4a9519a3977d74ff845f1c0b4ca2cb8f. at 1733286539363Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/a: creating writer at 1733286539364 (+1 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/a: appending metadata at 1733286539384 (+20 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/a: closing flushed file at 1733286539384Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/b: creating writer at 1733286539420 (+36 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/b: appending metadata at 1733286539441 (+21 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/b: closing flushed file at 1733286539441Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/c: creating writer at 1733286539873 (+432 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/c: appending metadata at 1733286539913 (+40 ms)Flushing 4a9519a3977d74ff845f1c0b4ca2cb8f/c: closing flushed file at 1733286539913Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cde9e77: reopening flushed file at 1733286540359 (+446 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bb9f85d: reopening flushed file at 1733286540370 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ec53cce: reopening flushed file at 1733286540381 (+11 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4a9519a3977d74ff845f1c0b4ca2cb8f in 1031ms, sequenceid=26, compaction requested=false; wal=null at 1733286540394 (+13 ms)Cleaning up temporary data from old regions at 1733286540402 (+8 ms)Running coprocessor post-open hooks at 1733286540415 (+13 ms)Region opened successfully at 1733286540416 (+1 ms) 2024-12-04T04:29:00,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=416 (was 409) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:46050 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:51274 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39924 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:51300 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39936 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:46092 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1112) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=764 (was 739) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4776 (was 4675) - AvailableMemoryMB LEAK? - 2024-12-04T04:29:00,452 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-04T04:29:00,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=416, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=764, ProcessCount=11, AvailableMemoryMB=4762 2024-12-04T04:29:00,470 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-04T04:29:00,490 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:00,492 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:00,493 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:29:00,496 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97630457, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-97630457, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:00,516 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97630457/hregion-97630457.1733286540497, exclude list is [], retry=0 2024-12-04T04:29:00,519 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:00,521 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:00,525 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:00,536 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97630457/hregion-97630457.1733286540497 2024-12-04T04:29:00,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:29:00,540 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9831d2a17cd162acb44883d59b41e4d4, NAME => 'testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:29:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741910_1088 (size=61) 2024-12-04T04:29:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741910_1088 (size=61) 2024-12-04T04:29:00,600 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:00,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741910_1088 (size=61) 2024-12-04T04:29:00,612 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,623 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9831d2a17cd162acb44883d59b41e4d4 columnFamilyName a 2024-12-04T04:29:00,623 DEBUG [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:00,633 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(327): Store=9831d2a17cd162acb44883d59b41e4d4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:00,633 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,643 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,643 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,672 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:29:00,702 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9831d2a17cd162acb44883d59b41e4d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60943285, jitterRate=-0.09187428653240204}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:29:00,703 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9831d2a17cd162acb44883d59b41e4d4: Writing region info on filesystem at 1733286540600Initializing all the Stores at 1733286540608 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286540608Cleaning up temporary data from old regions at 1733286540643 (+35 ms)Region opened successfully at 1733286540703 (+60 ms) 2024-12-04T04:29:00,703 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9831d2a17cd162acb44883d59b41e4d4, disabling compactions & flushes 2024-12-04T04:29:00,703 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,703 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,703 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. after waiting 0 ms 2024-12-04T04:29:00,704 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,708 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,708 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9831d2a17cd162acb44883d59b41e4d4: Waiting for close lock at 1733286540703Disabling compacts and flushes for region at 1733286540703Disabling writes for close at 1733286540703Writing region close event to WAL at 1733286540708 (+5 ms)Closed at 1733286540708 2024-12-04T04:29:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741909_1087 (size=93) 2024-12-04T04:29:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741909_1087 (size=93) 2024-12-04T04:29:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741909_1087 (size=93) 2024-12-04T04:29:00,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:29:00,733 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97630457:(num 1733286540497) 2024-12-04T04:29:00,733 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:00,736 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:00,749 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, exclude list is [], retry=0 2024-12-04T04:29:00,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:00,752 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:00,752 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:00,761 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 2024-12-04T04:29:00,764 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:29:00,764 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9831d2a17cd162acb44883d59b41e4d4, NAME => 'testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:00,765 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:00,765 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,765 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,768 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,769 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9831d2a17cd162acb44883d59b41e4d4 columnFamilyName a 2024-12-04T04:29:00,769 DEBUG [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:00,770 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(327): Store=9831d2a17cd162acb44883d59b41e4d4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:00,770 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,774 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,774 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,777 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:00,778 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9831d2a17cd162acb44883d59b41e4d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66611103, jitterRate=-0.007417216897010803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:29:00,778 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9831d2a17cd162acb44883d59b41e4d4: Writing region info on filesystem at 1733286540765Initializing all the Stores at 1733286540766 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286540766Cleaning up temporary data from old regions at 1733286540774 (+8 ms)Region opened successfully at 1733286540778 (+4 ms) 2024-12-04T04:29:00,800 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9831d2a17cd162acb44883d59b41e4d4, disabling compactions & flushes 2024-12-04T04:29:00,800 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,800 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,800 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. after waiting 0 ms 2024-12-04T04:29:00,800 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,801 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,801 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. 2024-12-04T04:29:00,801 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9831d2a17cd162acb44883d59b41e4d4: Waiting for close lock at 1733286540800Disabling compacts and flushes for region at 1733286540800Disabling writes for close at 1733286540800Writing region close event to WAL at 1733286540801 (+1 ms)Closed at 1733286540801 2024-12-04T04:29:00,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741911_1089 (size=838) 2024-12-04T04:29:00,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741911_1089 (size=838) 2024-12-04T04:29:00,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741911_1089 (size=838) 2024-12-04T04:29:00,833 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, size=838 (838bytes) 2024-12-04T04:29:00,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 2024-12-04T04:29:00,834 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 after 1ms 2024-12-04T04:29:00,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:00,837 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 took 4ms 2024-12-04T04:29:00,854 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 so closing down 2024-12-04T04:29:00,855 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:29:00,856 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733286540736.temp 2024-12-04T04:29:00,860 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp 2024-12-04T04:29:00,860 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:29:00,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741912_1090 (size=838) 2024-12-04T04:29:00,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741912_1090 (size=838) 2024-12-04T04:29:00,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741912_1090 (size=838) 2024-12-04T04:29:00,897 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-04T04:29:00,898 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp to hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 2024-12-04T04:29:00,899 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, size=838, length=838, corrupted=false, cancelled=false 2024-12-04T04:29:00,899 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, journal: Splitting hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, size=838 (838bytes) at 1733286540833Finishing writing output for hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 so closing down at 1733286540854 (+21 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp at 1733286540860 (+6 ms)3 split writer threads finished at 1733286540860Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733286540897 (+37 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000003-wal.1733286540736.temp to hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 at 1733286540899 (+2 ms)Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736, size=838, length=838, corrupted=false, cancelled=false at 1733286540899 2024-12-04T04:29:00,901 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286540736 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286540736 2024-12-04T04:29:00,902 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 2024-12-04T04:29:00,905 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:01,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:01,522 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:01,541 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286541522, exclude list is [], retry=0 2024-12-04T04:29:01,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:01,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:01,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:01,561 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286541522 2024-12-04T04:29:01,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9831d2a17cd162acb44883d59b41e4d4, NAME => 'testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,561 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,565 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,567 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9831d2a17cd162acb44883d59b41e4d4 columnFamilyName a 2024-12-04T04:29:01,568 DEBUG [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:01,569 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(327): Store=9831d2a17cd162acb44883d59b41e4d4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:01,569 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,570 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,572 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,572 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 2024-12-04T04:29:01,575 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:01,576 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 2024-12-04T04:29:01,577 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9831d2a17cd162acb44883d59b41e4d4 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-04T04:29:01,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/.tmp/a/02c49a47a11b4f16be76f49ca911445c is 79, key is testDatalossWhenInputError/a:x0/1733286540778/Put/seqid=0 2024-12-04T04:29:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741914_1092 (size=5808) 2024-12-04T04:29:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741914_1092 (size=5808) 2024-12-04T04:29:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741914_1092 (size=5808) 2024-12-04T04:29:01,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/.tmp/a/02c49a47a11b4f16be76f49ca911445c 2024-12-04T04:29:01,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/.tmp/a/02c49a47a11b4f16be76f49ca911445c as hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/a/02c49a47a11b4f16be76f49ca911445c 2024-12-04T04:29:01,686 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/a/02c49a47a11b4f16be76f49ca911445c, entries=10, sequenceid=12, filesize=5.7 K 2024-12-04T04:29:01,686 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9831d2a17cd162acb44883d59b41e4d4 in 110ms, sequenceid=12, compaction requested=false; wal=null 2024-12-04T04:29:01,689 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/0000000000000000012 2024-12-04T04:29:01,691 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,691 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,709 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,737 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T04:29:01,739 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9831d2a17cd162acb44883d59b41e4d4; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61277536, jitterRate=-0.08689355850219727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:29:01,739 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9831d2a17cd162acb44883d59b41e4d4: Writing region info on filesystem at 1733286541561Initializing all the Stores at 1733286541564 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286541564Obtaining lock to block concurrent updates at 1733286541577 (+13 ms)Preparing flush snapshotting stores in 9831d2a17cd162acb44883d59b41e4d4 at 1733286541577Finished memstore snapshotting testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733286541577Flushing stores of testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4. at 1733286541577Flushing 9831d2a17cd162acb44883d59b41e4d4/a: creating writer at 1733286541577Flushing 9831d2a17cd162acb44883d59b41e4d4/a: appending metadata at 1733286541595 (+18 ms)Flushing 9831d2a17cd162acb44883d59b41e4d4/a: closing flushed file at 1733286541595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d52c069: reopening flushed file at 1733286541663 (+68 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9831d2a17cd162acb44883d59b41e4d4 in 110ms, sequenceid=12, compaction requested=false; wal=null at 1733286541686 (+23 ms)Cleaning up temporary data from old regions at 1733286541691 (+5 ms)Region opened successfully at 1733286541739 (+48 ms) 2024-12-04T04:29:01,743 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9831d2a17cd162acb44883d59b41e4d4, NAME => 'testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:01,743 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733286540490.9831d2a17cd162acb44883d59b41e4d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:01,743 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,743 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,745 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,746 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9831d2a17cd162acb44883d59b41e4d4 columnFamilyName a 2024-12-04T04:29:01,746 DEBUG [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:01,775 DEBUG [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/a/02c49a47a11b4f16be76f49ca911445c 2024-12-04T04:29:01,775 INFO [StoreOpener-9831d2a17cd162acb44883d59b41e4d4-1 {}] regionserver.HStore(327): Store=9831d2a17cd162acb44883d59b41e4d4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:01,775 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,780 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,781 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,781 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,783 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9831d2a17cd162acb44883d59b41e4d4 2024-12-04T04:29:01,793 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testDatalossWhenInputError/9831d2a17cd162acb44883d59b41e4d4/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-04T04:29:01,808 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9831d2a17cd162acb44883d59b41e4d4; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69824441, jitterRate=0.04046525061130524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T04:29:01,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9831d2a17cd162acb44883d59b41e4d4: Writing region info on filesystem at 1733286541744Initializing all the Stores at 1733286541745 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286541745Cleaning up temporary data from old regions at 1733286541781 (+36 ms)Region opened successfully at 1733286541808 (+27 ms) 2024-12-04T04:29:01,842 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=426 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:40010 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:46156 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:51274 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:39936 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:51374 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:46092 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=823 (was 764) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4632 (was 4762) 2024-12-04T04:29:01,842 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-04T04:29:01,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=426, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=823, ProcessCount=11, AvailableMemoryMB=4583 2024-12-04T04:29:01,862 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-04T04:29:01,884 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:01,888 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:01,889 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:29:01,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55808103, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-55808103, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:01,914 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55808103/hregion-55808103.1733286541898, exclude list is [], retry=0 2024-12-04T04:29:01,916 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:01,920 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:01,922 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:01,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55808103/hregion-55808103.1733286541898 2024-12-04T04:29:01,931 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:29:01,931 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a3c6dcf3217d19c25e611cf17aad05c, NAME => 'testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:29:01,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741916_1094 (size=63) 2024-12-04T04:29:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741916_1094 (size=63) 2024-12-04T04:29:01,948 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741916_1094 (size=63) 2024-12-04T04:29:01,956 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,958 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName a 2024-12-04T04:29:01,958 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:01,958 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:01,958 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,959 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName b 2024-12-04T04:29:01,959 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:01,960 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:01,960 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,961 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName c 2024-12-04T04:29:01,961 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:01,962 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:01,963 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,964 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,964 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,965 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:01,966 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:01,967 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:29:01,968 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a3c6dcf3217d19c25e611cf17aad05c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71262638, jitterRate=0.061896055936813354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:01,968 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a3c6dcf3217d19c25e611cf17aad05c: Writing region info on filesystem at 1733286541948Initializing all the Stores at 1733286541952 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286541952Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286541956 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286541956Cleaning up temporary data from old regions at 1733286541964 (+8 ms)Region opened successfully at 1733286541968 (+4 ms) 2024-12-04T04:29:01,969 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a3c6dcf3217d19c25e611cf17aad05c, disabling compactions & flushes 2024-12-04T04:29:01,969 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:01,969 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:01,969 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. after waiting 0 ms 2024-12-04T04:29:01,969 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:01,972 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:01,972 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a3c6dcf3217d19c25e611cf17aad05c: Waiting for close lock at 1733286541969Disabling compacts and flushes for region at 1733286541969Disabling writes for close at 1733286541969Writing region close event to WAL at 1733286541972 (+3 ms)Closed at 1733286541972 2024-12-04T04:29:01,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741915_1093 (size=93) 2024-12-04T04:29:01,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741915_1093 (size=93) 2024-12-04T04:29:01,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741915_1093 (size=93) 2024-12-04T04:29:01,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:29:01,986 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55808103:(num 1733286541898) 2024-12-04T04:29:01,986 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:01,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:01,999 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, exclude list is [], retry=0 2024-12-04T04:29:02,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:02,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:02,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:02,008 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 2024-12-04T04:29:02,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361)] 2024-12-04T04:29:02,009 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a3c6dcf3217d19c25e611cf17aad05c, NAME => 'testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:02,009 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:02,009 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,009 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,011 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,011 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName a 2024-12-04T04:29:02,011 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:02,012 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:02,012 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,013 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName b 2024-12-04T04:29:02,013 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:02,014 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:02,014 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,015 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName c 2024-12-04T04:29:02,015 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:02,015 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:02,015 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,016 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,017 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,018 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,018 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:02,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:02,025 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a3c6dcf3217d19c25e611cf17aad05c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63552284, jitterRate=-0.0529971718788147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:02,026 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a3c6dcf3217d19c25e611cf17aad05c: Writing region info on filesystem at 1733286542010Initializing all the Stores at 1733286542010Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286542010Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286542011 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286542011Cleaning up temporary data from old regions at 1733286542018 (+7 ms)Region opened successfully at 1733286542026 (+8 ms) 2024-12-04T04:29:02,031 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733286542031/Put/seqid=0 2024-12-04T04:29:02,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741918_1096 (size=4875) 2024-12-04T04:29:02,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741918_1096 (size=4875) 2024-12-04T04:29:02,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741918_1096 (size=4875) 2024-12-04T04:29:02,057 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733286542057/Put/seqid=0 2024-12-04T04:29:02,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741919_1097 (size=4875) 2024-12-04T04:29:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741919_1097 (size=4875) 2024-12-04T04:29:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741919_1097 (size=4875) 2024-12-04T04:29:02,082 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733286542082/Put/seqid=0 2024-12-04T04:29:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741920_1098 (size=4875) 2024-12-04T04:29:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741920_1098 (size=4875) 2024-12-04T04:29:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741920_1098 (size=4875) 2024-12-04T04:29:02,116 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,121 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-04T04:29:02,121 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T04:29:02,122 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,126 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-04T04:29:02,126 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T04:29:02,126 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,130 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-04T04:29:02,130 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T04:29:02,130 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a3c6dcf3217d19c25e611cf17aad05c 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-04T04:29:02,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp/a/3ba9614380b34ebd8c4d947fd36145ff is 55, key is testCompactedBulkLoadedFiles/a:a/1733286542026/Put/seqid=0 2024-12-04T04:29:02,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741921_1099 (size=5107) 2024-12-04T04:29:02,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741921_1099 (size=5107) 2024-12-04T04:29:02,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741921_1099 (size=5107) 2024-12-04T04:29:02,155 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp/a/3ba9614380b34ebd8c4d947fd36145ff 2024-12-04T04:29:02,161 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp/a/3ba9614380b34ebd8c4d947fd36145ff as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff 2024-12-04T04:29:02,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff, entries=1, sequenceid=4, filesize=5.0 K 2024-12-04T04:29:02,167 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 6a3c6dcf3217d19c25e611cf17aad05c in 37ms, sequenceid=4, compaction requested=false 2024-12-04T04:29:02,167 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6a3c6dcf3217d19c25e611cf17aad05c: 2024-12-04T04:29:02,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ 2024-12-04T04:29:02,170 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ 2024-12-04T04:29:02,171 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ 2024-12-04T04:29:02,172 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile0 into 6a3c6dcf3217d19c25e611cf17aad05c/a as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ - updating store file list. 2024-12-04T04:29:02,176 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 961284d6c6a2452cacbd0e24546704e3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:02,177 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ into 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,177 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile0 into 6a3c6dcf3217d19c25e611cf17aad05c/a (new location: hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_) 2024-12-04T04:29:02,178 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile1 into 6a3c6dcf3217d19c25e611cf17aad05c/a as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ - updating store file list. 2024-12-04T04:29:02,183 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0566044355e148db9dd8a49023fc7abe_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:02,183 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ into 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,183 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile1 into 6a3c6dcf3217d19c25e611cf17aad05c/a (new location: hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_) 2024-12-04T04:29:02,185 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile2 into 6a3c6dcf3217d19c25e611cf17aad05c/a as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ - updating store file list. 2024-12-04T04:29:02,190 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2972a77600544d67b7c1bae60150a783_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:02,190 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ into 6a3c6dcf3217d19c25e611cf17aad05c/a 2024-12-04T04:29:02,190 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35021/hbase/testCompactedBulkLoadedFiles/hfile2 into 6a3c6dcf3217d19c25e611cf17aad05c/a (new location: hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_) 2024-12-04T04:29:02,198 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T04:29:02,199 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6a3c6dcf3217d19c25e611cf17aad05c/a is initiating major compaction (all files) 2024-12-04T04:29:02,199 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6a3c6dcf3217d19c25e611cf17aad05c/a in testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:02,199 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_] into tmpdir=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp, totalSize=19.3 K 2024-12-04T04:29:02,199 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3ba9614380b34ebd8c4d947fd36145ff, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733286542026 2024-12-04T04:29:02,200 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 961284d6c6a2452cacbd0e24546704e3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T04:29:02,200 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0566044355e148db9dd8a49023fc7abe_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T04:29:02,200 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2972a77600544d67b7c1bae60150a783_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T04:29:02,218 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp/a/d9d05aee87334104bc8b415b65a63853 is 55, key is testCompactedBulkLoadedFiles/a:a/1733286542026/Put/seqid=0 2024-12-04T04:29:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741922_1100 (size=6154) 2024-12-04T04:29:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741922_1100 (size=6154) 2024-12-04T04:29:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741922_1100 (size=6154) 2024-12-04T04:29:02,363 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-04T04:29:02,364 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-04T04:29:02,364 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-04T04:29:02,365 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-04T04:29:02,641 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/.tmp/a/d9d05aee87334104bc8b415b65a63853 as hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/d9d05aee87334104bc8b415b65a63853 2024-12-04T04:29:02,647 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 6a3c6dcf3217d19c25e611cf17aad05c/a of 6a3c6dcf3217d19c25e611cf17aad05c into d9d05aee87334104bc8b415b65a63853(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T04:29:02,647 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6a3c6dcf3217d19c25e611cf17aad05c: 2024-12-04T04:29:02,647 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T04:29:02,647 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T04:29:02,675 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, size=0 (0bytes) 2024-12-04T04:29:02,675 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 might be still open, length is 0 2024-12-04T04:29:02,675 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 2024-12-04T04:29:02,675 WARN [IPC Server handler 4 on default port 35021 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-04T04:29:02,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 after 0ms 2024-12-04T04:29:04,731 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T04:29:05,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:48238 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48238 dst: /127.0.0.1:43031 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43031 remote=/127.0.0.1:48238]. Total timeout mills is 60000, 57003 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:05,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:33290 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40387:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33290 dst: /127.0.0.1:40387 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:05,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:40708 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40708 dst: /127.0.0.1:38591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741917_1101 (size=1368) 2024-12-04T04:29:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741917_1101 (size=1368) 2024-12-04T04:29:05,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741917_1101 (size=1368) 2024-12-04T04:29:06,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 after 4001ms 2024-12-04T04:29:06,679 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:06,679 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 took 4005ms 2024-12-04T04:29:06,680 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987; continuing. 2024-12-04T04:29:06,680 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 so closing down 2024-12-04T04:29:06,681 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:29:06,681 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733286541987.temp 2024-12-04T04:29:06,683 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp 2024-12-04T04:29:06,683 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:29:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741923_1102 (size=635) 2024-12-04T04:29:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741923_1102 (size=635) 2024-12-04T04:29:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741923_1102 (size=635) 2024-12-04T04:29:06,689 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T04:29:06,690 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp to hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 2024-12-04T04:29:06,690 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T04:29:06,690 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, journal: Splitting hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, size=0 (0bytes) at 1733286542675Finishing writing output for hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 so closing down at 1733286546680 (+4005 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp at 1733286546683 (+3 ms)3 split writer threads finished at 1733286546683Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733286546689 (+6 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000003-wal.1733286541987.temp to hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 at 1733286546690 (+1 ms)Processed 5 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987, size=0, length=0, corrupted=false, cancelled=false at 1733286546690 2024-12-04T04:29:06,691 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286541987 2024-12-04T04:29:06,692 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 2024-12-04T04:29:06,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:06,693 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:06,704 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286546693, exclude list is [], retry=0 2024-12-04T04:29:06,707 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:06,707 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:06,707 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:06,709 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286546693 2024-12-04T04:29:06,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:29:06,709 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a3c6dcf3217d19c25e611cf17aad05c, NAME => 'testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:06,709 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:06,709 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,709 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,712 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,713 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName a 2024-12-04T04:29:06,713 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,718 DEBUG [StoreFileOpener-6a3c6dcf3217d19c25e611cf17aad05c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0566044355e148db9dd8a49023fc7abe_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:06,718 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ 2024-12-04T04:29:06,721 DEBUG [StoreFileOpener-6a3c6dcf3217d19c25e611cf17aad05c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2972a77600544d67b7c1bae60150a783_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:06,721 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ 2024-12-04T04:29:06,724 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff 2024-12-04T04:29:06,727 DEBUG [StoreFileOpener-6a3c6dcf3217d19c25e611cf17aad05c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 961284d6c6a2452cacbd0e24546704e3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T04:29:06,727 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ 2024-12-04T04:29:06,731 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/d9d05aee87334104bc8b415b65a63853 2024-12-04T04:29:06,731 WARN [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48529687 2024-12-04T04:29:06,731 WARN [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48529687 2024-12-04T04:29:06,731 WARN [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48529687 2024-12-04T04:29:06,731 WARN [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@48529687 2024-12-04T04:29:06,731 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_] to archive 2024-12-04T04:29:06,732 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T04:29:06,733 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ to hdfs://localhost:35021/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/0566044355e148db9dd8a49023fc7abe_SeqId_4_ 2024-12-04T04:29:06,734 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ to hdfs://localhost:35021/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/2972a77600544d67b7c1bae60150a783_SeqId_4_ 2024-12-04T04:29:06,735 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff to hdfs://localhost:35021/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/3ba9614380b34ebd8c4d947fd36145ff 2024-12-04T04:29:06,736 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ to hdfs://localhost:35021/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/a/961284d6c6a2452cacbd0e24546704e3_SeqId_4_ 2024-12-04T04:29:06,736 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,736 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,737 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName b 2024-12-04T04:29:06,737 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,737 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,738 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,738 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a3c6dcf3217d19c25e611cf17aad05c columnFamilyName c 2024-12-04T04:29:06,738 DEBUG [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,738 INFO [StoreOpener-6a3c6dcf3217d19c25e611cf17aad05c-1 {}] regionserver.HStore(327): Store=6a3c6dcf3217d19c25e611cf17aad05c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,738 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,739 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,740 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,741 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 2024-12-04T04:29:06,742 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:06,744 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 6a3c6dcf3217d19c25e611cf17aad05c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "6a3c6dcf3217d19c25e611cf17aad05c" family_name: "a" compaction_input: "3ba9614380b34ebd8c4d947fd36145ff" compaction_input: "961284d6c6a2452cacbd0e24546704e3_SeqId_4_" compaction_input: "0566044355e148db9dd8a49023fc7abe_SeqId_4_" compaction_input: "2972a77600544d67b7c1bae60150a783_SeqId_4_" compaction_output: "d9d05aee87334104bc8b415b65a63853" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-04T04:29:06,744 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-04T04:29:06,744 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 2024-12-04T04:29:06,745 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/0000000000000000008 2024-12-04T04:29:06,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,746 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,746 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:06,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a3c6dcf3217d19c25e611cf17aad05c 2024-12-04T04:29:06,749 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testCompactedBulkLoadedFiles/6a3c6dcf3217d19c25e611cf17aad05c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T04:29:06,750 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a3c6dcf3217d19c25e611cf17aad05c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70959072, jitterRate=0.0573725700378418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:06,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a3c6dcf3217d19c25e611cf17aad05c: Writing region info on filesystem at 1733286546709Initializing all the Stores at 1733286546710 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546710Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546712 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546712Cleaning up temporary data from old regions at 1733286546746 (+34 ms)Region opened successfully at 1733286546750 (+4 ms) 2024-12-04T04:29:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a3c6dcf3217d19c25e611cf17aad05c, disabling compactions & flushes 2024-12-04T04:29:06,752 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. after waiting 0 ms 2024-12-04T04:29:06,752 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:06,752 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733286541885.6a3c6dcf3217d19c25e611cf17aad05c. 2024-12-04T04:29:06,753 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a3c6dcf3217d19c25e611cf17aad05c: Waiting for close lock at 1733286546752Disabling compacts and flushes for region at 1733286546752Disabling writes for close at 1733286546752Writing region close event to WAL at 1733286546752Closed at 1733286546752 2024-12-04T04:29:06,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741924_1103 (size=93) 2024-12-04T04:29:06,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741924_1103 (size=93) 2024-12-04T04:29:06,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741924_1103 (size=93) 2024-12-04T04:29:06,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:29:06,756 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733286546693) 2024-12-04T04:29:06,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=434 (was 426) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:35021 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35021 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-744169814_22 at /127.0.0.1:33362 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-744169814_22 at /127.0.0.1:48312 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-744169814_22 at /127.0.0.1:40788 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=781 (was 823), ProcessCount=11 (was 11), AvailableMemoryMB=4619 (was 4583) - AvailableMemoryMB LEAK? - 2024-12-04T04:29:06,769 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-04T04:29:06,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=434, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=781, ProcessCount=11, AvailableMemoryMB=4618 2024-12-04T04:29:06,780 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-04T04:29:06,794 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:06,796 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T04:29:06,796 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T04:29:06,798 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66531835, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/hregion-66531835, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:06,811 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66531835/hregion-66531835.1733286546799, exclude list is [], retry=0 2024-12-04T04:29:06,813 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:06,814 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:06,814 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:06,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66531835/hregion-66531835.1733286546799 2024-12-04T04:29:06,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:29:06,817 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 26b237face91784e9990e2a65cdbf79d, NAME => 'testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35021/hbase 2024-12-04T04:29:06,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741926_1105 (size=67) 2024-12-04T04:29:06,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741926_1105 (size=67) 2024-12-04T04:29:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741926_1105 (size=67) 2024-12-04T04:29:06,828 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:06,832 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,833 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName a 2024-12-04T04:29:06,833 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,834 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,834 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,835 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName b 2024-12-04T04:29:06,835 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,835 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,836 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,837 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName c 2024-12-04T04:29:06,837 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,837 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,839 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,839 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,840 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:06,841 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,843 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T04:29:06,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 26b237face91784e9990e2a65cdbf79d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69970005, jitterRate=0.04263432323932648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 26b237face91784e9990e2a65cdbf79d: Writing region info on filesystem at 1733286546828Initializing all the Stores at 1733286546829 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546829Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546832 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546832Cleaning up temporary data from old regions at 1733286546839 (+7 ms)Region opened successfully at 1733286546844 (+5 ms) 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 26b237face91784e9990e2a65cdbf79d, disabling compactions & flushes 2024-12-04T04:29:06,844 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. after waiting 0 ms 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,844 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,844 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 26b237face91784e9990e2a65cdbf79d: Waiting for close lock at 1733286546844Disabling compacts and flushes for region at 1733286546844Disabling writes for close at 1733286546844Writing region close event to WAL at 1733286546844Closed at 1733286546844 2024-12-04T04:29:06,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741925_1104 (size=93) 2024-12-04T04:29:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741925_1104 (size=93) 2024-12-04T04:29:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741925_1104 (size=93) 2024-12-04T04:29:06,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:29:06,848 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66531835:(num 1733286546799) 2024-12-04T04:29:06,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:06,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:06,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, exclude list is [], retry=0 2024-12-04T04:29:06,864 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:06,864 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:06,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:06,866 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 2024-12-04T04:29:06,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:29:06,866 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 26b237face91784e9990e2a65cdbf79d, NAME => 'testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:06,867 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:06,867 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,867 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,868 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,869 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName a 2024-12-04T04:29:06,869 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,869 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,869 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,870 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName b 2024-12-04T04:29:06,870 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,870 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,871 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,871 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName c 2024-12-04T04:29:06,871 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:06,872 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:06,872 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,874 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,875 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:06,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:06,877 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 26b237face91784e9990e2a65cdbf79d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74723598, jitterRate=0.11346837878227234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:06,877 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 26b237face91784e9990e2a65cdbf79d: Writing region info on filesystem at 1733286546867Initializing all the Stores at 1733286546868 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546868Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546868Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286546868Cleaning up temporary data from old regions at 1733286546874 (+6 ms)Region opened successfully at 1733286546877 (+3 ms) 2024-12-04T04:29:06,883 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 26b237face91784e9990e2a65cdbf79d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T04:29:06,897 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/c3b34c39c26e4540b08a559e1e5c8722 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733286546877/Put/seqid=0 2024-12-04T04:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741928_1107 (size=5958) 2024-12-04T04:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741928_1107 (size=5958) 2024-12-04T04:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741928_1107 (size=5958) 2024-12-04T04:29:06,904 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/c3b34c39c26e4540b08a559e1e5c8722 2024-12-04T04:29:06,909 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/c3b34c39c26e4540b08a559e1e5c8722 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/c3b34c39c26e4540b08a559e1e5c8722 2024-12-04T04:29:06,915 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/c3b34c39c26e4540b08a559e1e5c8722, entries=10, sequenceid=13, filesize=5.8 K 2024-12-04T04:29:06,916 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 26b237face91784e9990e2a65cdbf79d in 33ms, sequenceid=13, compaction requested=false 2024-12-04T04:29:06,916 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 26b237face91784e9990e2a65cdbf79d: 2024-12-04T04:29:06,941 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 26b237face91784e9990e2a65cdbf79d, disabling compactions & flushes 2024-12-04T04:29:06,941 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,941 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,941 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. after waiting 0 ms 2024-12-04T04:29:06,941 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,942 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,942 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:06,942 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 26b237face91784e9990e2a65cdbf79d: Waiting for close lock at 1733286546941Disabling compacts and flushes for region at 1733286546941Disabling writes for close at 1733286546941Writing region close event to WAL at 1733286546942 (+1 ms)Closed at 1733286546942 2024-12-04T04:29:06,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741927_1106 (size=2805) 2024-12-04T04:29:06,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741927_1106 (size=2805) 2024-12-04T04:29:06,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741927_1106 (size=2805) 2024-12-04T04:29:06,966 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, size=2.7 K (2805bytes) 2024-12-04T04:29:06,966 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 2024-12-04T04:29:06,967 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 after 1ms 2024-12-04T04:29:06,970 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:06,970 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 took 4ms 2024-12-04T04:29:06,972 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 so closing down 2024-12-04T04:29:06,972 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:29:06,974 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733286546850.temp 2024-12-04T04:29:06,975 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp 2024-12-04T04:29:06,976 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:29:06,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741929_1108 (size=2312) 2024-12-04T04:29:06,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741929_1108 (size=2312) 2024-12-04T04:29:06,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741929_1108 (size=2312) 2024-12-04T04:29:06,997 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T04:29:06,999 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 2024-12-04T04:29:06,999 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-04T04:29:06,999 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, journal: Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, size=2.7 K (2805bytes) at 1733286546966Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 so closing down at 1733286546972 (+6 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp at 1733286546975 (+3 ms)3 split writer threads finished at 1733286546976 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733286546997 (+21 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000003-wal.1733286546850.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 at 1733286546999 (+2 ms)Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733286546999 2024-12-04T04:29:07,001 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286546850 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286546850 2024-12-04T04:29:07,002 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 2024-12-04T04:29:07,002 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:07,004 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:07,024 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, exclude list is [], retry=0 2024-12-04T04:29:07,027 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:07,027 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:07,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:07,045 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 2024-12-04T04:29:07,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:42201:42201),(127.0.0.1/127.0.0.1:35321:35321)] 2024-12-04T04:29:07,045 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 26b237face91784e9990e2a65cdbf79d, NAME => 'testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T04:29:07,045 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:07,045 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,045 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,047 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,048 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName a 2024-12-04T04:29:07,049 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:07,054 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/c3b34c39c26e4540b08a559e1e5c8722 2024-12-04T04:29:07,054 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:07,055 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,057 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName b 2024-12-04T04:29:07,057 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:07,057 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:07,058 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,059 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName c 2024-12-04T04:29:07,059 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:07,059 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:07,060 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,060 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,063 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 2024-12-04T04:29:07,065 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:07,066 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 2024-12-04T04:29:07,066 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 26b237face91784e9990e2a65cdbf79d 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-04T04:29:07,081 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8635260ea380478c80263c908740ffa5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733286546916/Put/seqid=0 2024-12-04T04:29:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741931_1110 (size=5958) 2024-12-04T04:29:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741931_1110 (size=5958) 2024-12-04T04:29:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741931_1110 (size=5958) 2024-12-04T04:29:07,490 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8635260ea380478c80263c908740ffa5 2024-12-04T04:29:07,509 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/da3d59f1eb124ebf92ab9eecca11319f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733286546927/Put/seqid=0 2024-12-04T04:29:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741932_1111 (size=5958) 2024-12-04T04:29:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741932_1111 (size=5958) 2024-12-04T04:29:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741932_1111 (size=5958) 2024-12-04T04:29:07,516 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/da3d59f1eb124ebf92ab9eecca11319f 2024-12-04T04:29:07,521 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8635260ea380478c80263c908740ffa5 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/b/8635260ea380478c80263c908740ffa5 2024-12-04T04:29:07,526 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/b/8635260ea380478c80263c908740ffa5, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T04:29:07,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/da3d59f1eb124ebf92ab9eecca11319f as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/c/da3d59f1eb124ebf92ab9eecca11319f 2024-12-04T04:29:07,532 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/c/da3d59f1eb124ebf92ab9eecca11319f, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T04:29:07,533 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 26b237face91784e9990e2a65cdbf79d in 467ms, sequenceid=35, compaction requested=false; wal=null 2024-12-04T04:29:07,533 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000035 2024-12-04T04:29:07,535 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,535 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,535 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:07,537 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:07,539 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-04T04:29:07,540 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 26b237face91784e9990e2a65cdbf79d; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71873914, jitterRate=0.07100477814674377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:07,540 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 26b237face91784e9990e2a65cdbf79d: Writing region info on filesystem at 1733286547046Initializing all the Stores at 1733286547047 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286547047Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286547047Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286547047Obtaining lock to block concurrent updates at 1733286547066 (+19 ms)Preparing flush snapshotting stores in 26b237face91784e9990e2a65cdbf79d at 1733286547066Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733286547067 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. at 1733286547067Flushing 26b237face91784e9990e2a65cdbf79d/b: creating writer at 1733286547067Flushing 26b237face91784e9990e2a65cdbf79d/b: appending metadata at 1733286547081 (+14 ms)Flushing 26b237face91784e9990e2a65cdbf79d/b: closing flushed file at 1733286547081Flushing 26b237face91784e9990e2a65cdbf79d/c: creating writer at 1733286547496 (+415 ms)Flushing 26b237face91784e9990e2a65cdbf79d/c: appending metadata at 1733286547509 (+13 ms)Flushing 26b237face91784e9990e2a65cdbf79d/c: closing flushed file at 1733286547509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@90780e4: reopening flushed file at 1733286547521 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62368e04: reopening flushed file at 1733286547526 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 26b237face91784e9990e2a65cdbf79d in 467ms, sequenceid=35, compaction requested=false; wal=null at 1733286547533 (+7 ms)Cleaning up temporary data from old regions at 1733286547535 (+2 ms)Region opened successfully at 1733286547540 (+5 ms) 2024-12-04T04:29:07,597 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, size=0 (0bytes) 2024-12-04T04:29:07,597 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 might be still open, length is 0 2024-12-04T04:29:07,597 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 2024-12-04T04:29:07,597 WARN [IPC Server handler 0 on default port 35021 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-04T04:29:07,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 after 1ms 2024-12-04T04:29:08,148 INFO [master/67abd3a0ee76:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T04:29:08,148 INFO [master/67abd3a0ee76:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T04:29:08,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:33428 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40387:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33428 dst: /127.0.0.1:40387 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40387 remote=/127.0.0.1:33428]. Total timeout mills is 60000, 58918 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:08,646 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:40846 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38591:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40846 dst: /127.0.0.1:38591 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:08,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1523800604_22 at /127.0.0.1:48376 [Receiving block BP-90967970-172.17.0.3-1733286495978:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48376 dst: /127.0.0.1:43031 java.io.InterruptedIOException: Interrupted receiveBlock at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:08,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741930_1112 (size=2304) 2024-12-04T04:29:08,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741930_1112 (size=2304) 2024-12-04T04:29:11,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 after 4001ms 2024-12-04T04:29:11,601 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:11,601 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 took 4004ms 2024-12-04T04:29:11,603 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005; continuing. 2024-12-04T04:29:11,603 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 so closing down 2024-12-04T04:29:11,603 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T04:29:11,604 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733286547005.temp 2024-12-04T04:29:11,605 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp 2024-12-04T04:29:11,606 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T04:29:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741933_1113 (size=2312) 2024-12-04T04:29:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741933_1113 (size=2312) 2024-12-04T04:29:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741933_1113 (size=2312) 2024-12-04T04:29:11,612 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T04:29:11,613 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 2024-12-04T04:29:11,613 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T04:29:11,613 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, journal: Splitting hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, size=0 (0bytes) at 1733286547597Finishing writing output for hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 so closing down at 1733286551603 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp at 1733286551605 (+2 ms)3 split writer threads finished at 1733286551606 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733286551612 (+6 ms)Rename recovered edits hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000037-wal.1733286547005.temp to hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 at 1733286551613 (+1 ms)Processed 30 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005, size=0, length=0, corrupted=false, cancelled=false at 1733286551613 2024-12-04T04:29:11,615 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 to hdfs://localhost:35021/hbase/oldWALs/wal.1733286547005 2024-12-04T04:29:11,616 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 2024-12-04T04:29:11,617 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T04:29:11,618 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35021/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793, archiveDir=hdfs://localhost:35021/hbase/oldWALs, maxLogs=32 2024-12-04T04:29:11,638 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286551619, exclude list is [], retry=0 2024-12-04T04:29:11,665 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40387,DS-9d637861-4190-4908-8983-0774d7a0d00a,DISK] 2024-12-04T04:29:11,665 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43031,DS-31ae20d1-2601-4459-b770-2152aaba8d02,DISK] 2024-12-04T04:29:11,665 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38591,DS-603f0531-f499-4df5-bf23-553f02acec2f,DISK] 2024-12-04T04:29:11,670 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286551619 2024-12-04T04:29:11,671 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41361:41361),(127.0.0.1/127.0.0.1:35321:35321),(127.0.0.1/127.0.0.1:42201:42201)] 2024-12-04T04:29:11,672 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T04:29:11,673 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,674 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName a 2024-12-04T04:29:11,674 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:11,680 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/c3b34c39c26e4540b08a559e1e5c8722 2024-12-04T04:29:11,680 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:11,680 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,685 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName b 2024-12-04T04:29:11,685 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:11,704 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/b/8635260ea380478c80263c908740ffa5 2024-12-04T04:29:11,704 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:11,704 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,705 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26b237face91784e9990e2a65cdbf79d columnFamilyName c 2024-12-04T04:29:11,705 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T04:29:11,721 DEBUG [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/c/da3d59f1eb124ebf92ab9eecca11319f 2024-12-04T04:29:11,721 INFO [StoreOpener-26b237face91784e9990e2a65cdbf79d-1 {}] regionserver.HStore(327): Store=26b237face91784e9990e2a65cdbf79d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T04:29:11,721 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,722 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,724 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:11,724 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 2024-12-04T04:29:11,726 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T04:29:11,730 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 2024-12-04T04:29:11,730 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 26b237face91784e9990e2a65cdbf79d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T04:29:11,765 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/ab5fbb05428344ba8ba6127c350e2f1a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733286547546/Put/seqid=0 2024-12-04T04:29:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741935_1115 (size=5958) 2024-12-04T04:29:11,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741935_1115 (size=5958) 2024-12-04T04:29:11,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741935_1115 (size=5958) 2024-12-04T04:29:11,786 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/ab5fbb05428344ba8ba6127c350e2f1a 2024-12-04T04:29:11,813 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8559488deae3465785d2c0369e117610 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733286547552/Put/seqid=0 2024-12-04T04:29:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741936_1116 (size=5958) 2024-12-04T04:29:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741936_1116 (size=5958) 2024-12-04T04:29:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741936_1116 (size=5958) 2024-12-04T04:29:11,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8559488deae3465785d2c0369e117610 2024-12-04T04:29:11,910 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/30590d00776c44c68722c11339e9b5aa is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733286547558/Put/seqid=0 2024-12-04T04:29:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741937_1117 (size=5958) 2024-12-04T04:29:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741937_1117 (size=5958) 2024-12-04T04:29:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741937_1117 (size=5958) 2024-12-04T04:29:11,975 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/30590d00776c44c68722c11339e9b5aa 2024-12-04T04:29:11,996 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/a/ab5fbb05428344ba8ba6127c350e2f1a as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/ab5fbb05428344ba8ba6127c350e2f1a 2024-12-04T04:29:12,013 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/a/ab5fbb05428344ba8ba6127c350e2f1a, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T04:29:12,015 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/b/8559488deae3465785d2c0369e117610 as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/b/8559488deae3465785d2c0369e117610 2024-12-04T04:29:12,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/b/8559488deae3465785d2c0369e117610, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T04:29:12,045 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/.tmp/c/30590d00776c44c68722c11339e9b5aa as hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/c/30590d00776c44c68722c11339e9b5aa 2024-12-04T04:29:12,063 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/c/30590d00776c44c68722c11339e9b5aa, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T04:29:12,063 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 26b237face91784e9990e2a65cdbf79d in 333ms, sequenceid=66, compaction requested=false; wal=null 2024-12-04T04:29:12,065 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/0000000000000000066 2024-12-04T04:29:12,078 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:12,078 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:12,083 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T04:29:12,096 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 26b237face91784e9990e2a65cdbf79d 2024-12-04T04:29:12,109 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/hbase/data/default/testReplayEditsWrittenViaHRegion/26b237face91784e9990e2a65cdbf79d/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-04T04:29:12,120 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 26b237face91784e9990e2a65cdbf79d; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74563207, jitterRate=0.11107836663722992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T04:29:12,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 26b237face91784e9990e2a65cdbf79d: Writing region info on filesystem at 1733286551672Initializing all the Stores at 1733286551673 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286551673Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286551673Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733286551673Obtaining lock to block concurrent updates at 1733286551730 (+57 ms)Preparing flush snapshotting stores in 26b237face91784e9990e2a65cdbf79d at 1733286551730Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733286551730Flushing stores of testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. at 1733286551730Flushing 26b237face91784e9990e2a65cdbf79d/a: creating writer at 1733286551730Flushing 26b237face91784e9990e2a65cdbf79d/a: appending metadata at 1733286551764 (+34 ms)Flushing 26b237face91784e9990e2a65cdbf79d/a: closing flushed file at 1733286551764Flushing 26b237face91784e9990e2a65cdbf79d/b: creating writer at 1733286551796 (+32 ms)Flushing 26b237face91784e9990e2a65cdbf79d/b: appending metadata at 1733286551812 (+16 ms)Flushing 26b237face91784e9990e2a65cdbf79d/b: closing flushed file at 1733286551812Flushing 26b237face91784e9990e2a65cdbf79d/c: creating writer at 1733286551876 (+64 ms)Flushing 26b237face91784e9990e2a65cdbf79d/c: appending metadata at 1733286551909 (+33 ms)Flushing 26b237face91784e9990e2a65cdbf79d/c: closing flushed file at 1733286551909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43087bf6: reopening flushed file at 1733286551987 (+78 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48259977: reopening flushed file at 1733286552014 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17914c29: reopening flushed file at 1733286552041 (+27 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 26b237face91784e9990e2a65cdbf79d in 333ms, sequenceid=66, compaction requested=false; wal=null at 1733286552063 (+22 ms)Cleaning up temporary data from old regions at 1733286552078 (+15 ms)Region opened successfully at 1733286552120 (+42 ms) 2024-12-04T04:29:12,154 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 26b237face91784e9990e2a65cdbf79d, disabling compactions & flushes 2024-12-04T04:29:12,154 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:12,154 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:12,155 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. after waiting 0 ms 2024-12-04T04:29:12,155 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:12,167 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733286546794.26b237face91784e9990e2a65cdbf79d. 2024-12-04T04:29:12,167 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 26b237face91784e9990e2a65cdbf79d: Waiting for close lock at 1733286552154Disabling compacts and flushes for region at 1733286552154Disabling writes for close at 1733286552155 (+1 ms)Writing region close event to WAL at 1733286552167 (+12 ms)Closed at 1733286552167 2024-12-04T04:29:12,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741934_1114 (size=93) 2024-12-04T04:29:12,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741934_1114 (size=93) 2024-12-04T04:29:12,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741934_1114 (size=93) 2024-12-04T04:29:12,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T04:29:12,229 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733286551619) 2024-12-04T04:29:12,259 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=441 (was 434) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1698716709_22 at /127.0.0.1:40876 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1698716709_22 at /127.0.0.1:33476 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1396088596) connection to localhost/127.0.0.1:35021 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35021 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1698716709_22 at /127.0.0.1:48418 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=767 (was 781), ProcessCount=11 (was 11), AvailableMemoryMB=4619 (was 4618) - AvailableMemoryMB LEAK? - 2024-12-04T04:29:12,259 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-12-04T04:29:12,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T04:29:12,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T04:29:12,260 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T04:29:12,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T04:29:12,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T04:29:12,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1950715143, stopped=false 2024-12-04T04:29:12,261 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=67abd3a0ee76,33125,1733286501754 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:29:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:29:12,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T04:29:12,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:29:12,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:29:12,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T04:29:12,280 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T04:29:12,280 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T04:29:12,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,280 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '67abd3a0ee76,40991,1733286503128' ***** 2024-12-04T04:29:12,280 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T04:29:12,280 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T04:29:12,281 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(959): stopping server 67abd3a0ee76,40991,1733286503128 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;67abd3a0ee76:40991. 2024-12-04T04:29:12,281 DEBUG [RS:0;67abd3a0ee76:40991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T04:29:12,281 DEBUG [RS:0;67abd3a0ee76:40991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,281 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(976): stopping server 67abd3a0ee76,40991,1733286503128; all regions closed. 2024-12-04T04:29:12,282 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '67abd3a0ee76,35953,1733286503448' ***** 2024-12-04T04:29:12,282 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T04:29:12,282 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T04:29:12,282 INFO [RS:2;67abd3a0ee76:35953 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T04:29:12,282 INFO [RS:2;67abd3a0ee76:35953 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T04:29:12,282 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:29:12,283 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(959): stopping server 67abd3a0ee76,35953,1733286503448 2024-12-04T04:29:12,283 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T04:29:12,283 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T04:29:12,284 INFO [RS:2;67abd3a0ee76:35953 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;67abd3a0ee76:35953. 2024-12-04T04:29:12,284 DEBUG [RS:2;67abd3a0ee76:35953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T04:29:12,284 DEBUG [RS:2;67abd3a0ee76:35953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,284 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c8da809e64ee3e582bbcab19c4b44fb, disabling compactions & flushes 2024-12-04T04:29:12,284 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T04:29:12,284 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T04:29:12,284 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:29:12,284 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T04:29:12,284 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T04:29:12,284 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:29:12,284 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. after waiting 0 ms 2024-12-04T04:29:12,284 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:29:12,287 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T04:29:12,287 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1325): Online Regions={3c8da809e64ee3e582bbcab19c4b44fb=testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T04:29:12,287 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3c8da809e64ee3e582bbcab19c4b44fb 2024-12-04T04:29:12,287 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T04:29:12,287 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T04:29:12,287 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T04:29:12,287 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T04:29:12,287 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T04:29:12,287 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-04T04:29:12,289 INFO [regionserver/67abd3a0ee76:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:29:12,296 INFO [regionserver/67abd3a0ee76:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:29:12,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741834_1010 (size=2054) 2024-12-04T04:29:12,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741834_1010 (size=2054) 2024-12-04T04:29:12,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741834_1010 (size=2054) 2024-12-04T04:29:12,382 DEBUG [RS:0;67abd3a0ee76:40991 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs 2024-12-04T04:29:12,382 INFO [RS:0;67abd3a0ee76:40991 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 67abd3a0ee76%2C40991%2C1733286503128:(num 1733286506176) 2024-12-04T04:29:12,382 DEBUG [RS:0;67abd3a0ee76:40991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:12,383 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:29:12,385 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T04:29:12,385 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3c8da809e64ee3e582bbcab19c4b44fb/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.ChoreService(370): Chore service for: regionserver/67abd3a0ee76:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T04:29:12,389 INFO [RS:0;67abd3a0ee76:40991 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40991 2024-12-04T04:29:12,390 INFO [regionserver/67abd3a0ee76:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T04:29:12,390 INFO [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:29:12,390 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c8da809e64ee3e582bbcab19c4b44fb: Waiting for close lock at 1733286552284Running coprocessor pre-close hooks at 1733286552284Disabling compacts and flushes for region at 1733286552284Disabling writes for close at 1733286552284Writing region close event to WAL at 1733286552320 (+36 ms)Running coprocessor post-close hooks at 1733286552390 (+70 ms)Closed at 1733286552390 2024-12-04T04:29:12,391 DEBUG [RS_CLOSE_REGION-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb. 2024-12-04T04:29:12,413 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/info/6e15b7154d6f4b7781898df7524f17f6 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733286522575.3c8da809e64ee3e582bbcab19c4b44fb./info:regioninfo/1733286527138/Put/seqid=0 2024-12-04T04:29:12,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T04:29:12,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/67abd3a0ee76,40991,1733286503128 2024-12-04T04:29:12,436 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T04:29:12,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741938_1118 (size=8243) 2024-12-04T04:29:12,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741938_1118 (size=8243) 2024-12-04T04:29:12,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741938_1118 (size=8243) 2024-12-04T04:29:12,449 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/info/6e15b7154d6f4b7781898df7524f17f6 2024-12-04T04:29:12,459 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [67abd3a0ee76,40991,1733286503128] 2024-12-04T04:29:12,471 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/67abd3a0ee76,40991,1733286503128 already deleted, retry=false 2024-12-04T04:29:12,471 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 67abd3a0ee76,40991,1733286503128 expired; onlineServers=1 2024-12-04T04:29:12,487 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T04:29:12,539 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/ns/9f756fbfce064a09a9c3b63043c6c8e8 is 43, key is default/ns:d/1733286507216/Put/seqid=0 2024-12-04T04:29:12,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:12,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40991-0x1019e9915430001, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:12,560 INFO [RS:0;67abd3a0ee76:40991 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T04:29:12,560 INFO [RS:0;67abd3a0ee76:40991 {}] regionserver.HRegionServer(1031): Exiting; stopping=67abd3a0ee76,40991,1733286503128; zookeeper connection closed. 2024-12-04T04:29:12,562 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73eaaa09 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73eaaa09 2024-12-04T04:29:12,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741939_1119 (size=5153) 2024-12-04T04:29:12,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741939_1119 (size=5153) 2024-12-04T04:29:12,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741939_1119 (size=5153) 2024-12-04T04:29:12,616 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/ns/9f756fbfce064a09a9c3b63043c6c8e8 2024-12-04T04:29:12,687 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T04:29:12,702 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/table/06dfe3ac19224121ac53cf1fc55f2176 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733286523027/Put/seqid=0 2024-12-04T04:29:12,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741940_1120 (size=5431) 2024-12-04T04:29:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741940_1120 (size=5431) 2024-12-04T04:29:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741940_1120 (size=5431) 2024-12-04T04:29:12,888 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T04:29:13,055 INFO [regionserver/67abd3a0ee76:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T04:29:13,056 INFO [regionserver/67abd3a0ee76:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T04:29:13,088 DEBUG [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T04:29:13,134 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/table/06dfe3ac19224121ac53cf1fc55f2176 2024-12-04T04:29:13,142 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/info/6e15b7154d6f4b7781898df7524f17f6 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/info/6e15b7154d6f4b7781898df7524f17f6 2024-12-04T04:29:13,149 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/info/6e15b7154d6f4b7781898df7524f17f6, entries=18, sequenceid=21, filesize=8.0 K 2024-12-04T04:29:13,150 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/ns/9f756fbfce064a09a9c3b63043c6c8e8 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/ns/9f756fbfce064a09a9c3b63043c6c8e8 2024-12-04T04:29:13,156 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/ns/9f756fbfce064a09a9c3b63043c6c8e8, entries=2, sequenceid=21, filesize=5.0 K 2024-12-04T04:29:13,157 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/.tmp/table/06dfe3ac19224121ac53cf1fc55f2176 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/table/06dfe3ac19224121ac53cf1fc55f2176 2024-12-04T04:29:13,163 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/table/06dfe3ac19224121ac53cf1fc55f2176, entries=2, sequenceid=21, filesize=5.3 K 2024-12-04T04:29:13,164 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 877ms, sequenceid=21, compaction requested=false 2024-12-04T04:29:13,168 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T04:29:13,169 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T04:29:13,169 INFO [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T04:29:13,169 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733286552287Running coprocessor pre-close hooks at 1733286552287Disabling compacts and flushes for region at 1733286552287Disabling writes for close at 1733286552287Obtaining lock to block concurrent updates at 1733286552287Preparing flush snapshotting stores in 1588230740 at 1733286552287Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733286552288 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733286552296 (+8 ms)Flushing 1588230740/info: creating writer at 1733286552298 (+2 ms)Flushing 1588230740/info: appending metadata at 1733286552413 (+115 ms)Flushing 1588230740/info: closing flushed file at 1733286552413Flushing 1588230740/ns: creating writer at 1733286552468 (+55 ms)Flushing 1588230740/ns: appending metadata at 1733286552539 (+71 ms)Flushing 1588230740/ns: closing flushed file at 1733286552539Flushing 1588230740/table: creating writer at 1733286552677 (+138 ms)Flushing 1588230740/table: appending metadata at 1733286552702 (+25 ms)Flushing 1588230740/table: closing flushed file at 1733286552702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@402dde0d: reopening flushed file at 1733286553141 (+439 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cfa88f5: reopening flushed file at 1733286553149 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c3dc95a: reopening flushed file at 1733286553156 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 877ms, sequenceid=21, compaction requested=false at 1733286553164 (+8 ms)Writing region close event to WAL at 1733286553165 (+1 ms)Running coprocessor post-close hooks at 1733286553169 (+4 ms)Closed at 1733286553169 2024-12-04T04:29:13,170 DEBUG [RS_CLOSE_META-regionserver/67abd3a0ee76:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T04:29:13,288 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(976): stopping server 67abd3a0ee76,35953,1733286503448; all regions closed. 2024-12-04T04:29:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741836_1012 (size=3783) 2024-12-04T04:29:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741836_1012 (size=3783) 2024-12-04T04:29:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741836_1012 (size=3783) 2024-12-04T04:29:13,293 DEBUG [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs 2024-12-04T04:29:13,293 INFO [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 67abd3a0ee76%2C35953%2C1733286503448.meta:.meta(num 1733286506837) 2024-12-04T04:29:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741833_1009 (size=841) 2024-12-04T04:29:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741833_1009 (size=841) 2024-12-04T04:29:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741833_1009 (size=841) 2024-12-04T04:29:13,300 DEBUG [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/oldWALs 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 67abd3a0ee76%2C35953%2C1733286503448:(num 1733286506176) 2024-12-04T04:29:13,301 DEBUG [RS:2;67abd3a0ee76:35953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.ChoreService(370): Chore service for: regionserver/67abd3a0ee76:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T04:29:13,301 INFO [RS:2;67abd3a0ee76:35953 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35953 2024-12-04T04:29:13,302 INFO [regionserver/67abd3a0ee76:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T04:29:13,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T04:29:13,382 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T04:29:13,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/67abd3a0ee76,35953,1733286503448 2024-12-04T04:29:13,456 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [67abd3a0ee76,35953,1733286503448] 2024-12-04T04:29:13,506 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/67abd3a0ee76,35953,1733286503448 already deleted, retry=false 2024-12-04T04:29:13,507 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 67abd3a0ee76,35953,1733286503448 expired; onlineServers=0 2024-12-04T04:29:13,507 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '67abd3a0ee76,33125,1733286501754' ***** 2024-12-04T04:29:13,507 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T04:29:13,507 INFO [M:0;67abd3a0ee76:33125 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T04:29:13,507 INFO [M:0;67abd3a0ee76:33125 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T04:29:13,507 DEBUG [M:0;67abd3a0ee76:33125 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T04:29:13,507 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T04:29:13,507 DEBUG [M:0;67abd3a0ee76:33125 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T04:29:13,507 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.large.0-1733286505717 {}] cleaner.HFileCleaner(306): Exit Thread[master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.large.0-1733286505717,5,FailOnTimeoutGroup] 2024-12-04T04:29:13,507 DEBUG [master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.small.0-1733286505718 {}] cleaner.HFileCleaner(306): Exit Thread[master/67abd3a0ee76:0:becomeActiveMaster-HFileCleaner.small.0-1733286505718,5,FailOnTimeoutGroup] 2024-12-04T04:29:13,508 INFO [M:0;67abd3a0ee76:33125 {}] hbase.ChoreService(370): Chore service for: master/67abd3a0ee76:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T04:29:13,508 INFO [M:0;67abd3a0ee76:33125 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T04:29:13,508 DEBUG [M:0;67abd3a0ee76:33125 {}] master.HMaster(1795): Stopping service threads 2024-12-04T04:29:13,508 INFO [M:0;67abd3a0ee76:33125 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T04:29:13,508 INFO [M:0;67abd3a0ee76:33125 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T04:29:13,508 INFO [M:0;67abd3a0ee76:33125 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T04:29:13,509 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T04:29:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T04:29:13,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T04:29:13,530 DEBUG [M:0;67abd3a0ee76:33125 {}] zookeeper.ZKUtil(347): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T04:29:13,530 WARN [M:0;67abd3a0ee76:33125 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T04:29:13,531 INFO [M:0;67abd3a0ee76:33125 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/.lastflushedseqids 2024-12-04T04:29:13,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741941_1121 (size=138) 2024-12-04T04:29:13,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741941_1121 (size=138) 2024-12-04T04:29:13,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741941_1121 (size=138) 2024-12-04T04:29:13,546 INFO [M:0;67abd3a0ee76:33125 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T04:29:13,546 INFO [M:0;67abd3a0ee76:33125 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T04:29:13,546 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T04:29:13,546 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:29:13,546 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:29:13,546 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T04:29:13,547 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:29:13,547 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-04T04:29:13,556 INFO [RS:2;67abd3a0ee76:35953 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T04:29:13,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:13,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35953-0x1019e9915430003, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:13,557 INFO [RS:2;67abd3a0ee76:35953 {}] regionserver.HRegionServer(1031): Exiting; stopping=67abd3a0ee76,35953,1733286503448; zookeeper connection closed. 2024-12-04T04:29:13,557 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51275370 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51275370 2024-12-04T04:29:13,560 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T04:29:13,566 DEBUG [M:0;67abd3a0ee76:33125 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f083739a80234812a20ff6530905754e is 82, key is hbase:meta,,1/info:regioninfo/1733286506914/Put/seqid=0 2024-12-04T04:29:13,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741942_1122 (size=5672) 2024-12-04T04:29:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741942_1122 (size=5672) 2024-12-04T04:29:13,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741942_1122 (size=5672) 2024-12-04T04:29:13,575 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f083739a80234812a20ff6530905754e 2024-12-04T04:29:13,608 DEBUG [M:0;67abd3a0ee76:33125 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bc9810c6da548f9a2230f0b62d813f9 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733286523036/Put/seqid=0 2024-12-04T04:29:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741943_1123 (size=7756) 2024-12-04T04:29:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741943_1123 (size=7756) 2024-12-04T04:29:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741943_1123 (size=7756) 2024-12-04T04:29:14,033 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bc9810c6da548f9a2230f0b62d813f9 2024-12-04T04:29:14,037 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bc9810c6da548f9a2230f0b62d813f9 2024-12-04T04:29:14,061 DEBUG [M:0;67abd3a0ee76:33125 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3d834d51bb469485f350d5c225fbc4 is 69, key is 67abd3a0ee76,35953,1733286503448/rs:state/1733286505823/Put/seqid=0 2024-12-04T04:29:14,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741944_1124 (size=5440) 2024-12-04T04:29:14,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741944_1124 (size=5440) 2024-12-04T04:29:14,101 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3d834d51bb469485f350d5c225fbc4 2024-12-04T04:29:14,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741944_1124 (size=5440) 2024-12-04T04:29:14,107 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc3d834d51bb469485f350d5c225fbc4 2024-12-04T04:29:14,108 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f083739a80234812a20ff6530905754e as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f083739a80234812a20ff6530905754e 2024-12-04T04:29:14,117 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f083739a80234812a20ff6530905754e, entries=8, sequenceid=168, filesize=5.5 K 2024-12-04T04:29:14,119 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bc9810c6da548f9a2230f0b62d813f9 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bc9810c6da548f9a2230f0b62d813f9 2024-12-04T04:29:14,129 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bc9810c6da548f9a2230f0b62d813f9 2024-12-04T04:29:14,129 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bc9810c6da548f9a2230f0b62d813f9, entries=17, sequenceid=168, filesize=7.6 K 2024-12-04T04:29:14,131 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3d834d51bb469485f350d5c225fbc4 as hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc3d834d51bb469485f350d5c225fbc4 2024-12-04T04:29:14,138 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc3d834d51bb469485f350d5c225fbc4 2024-12-04T04:29:14,138 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35021/user/jenkins/test-data/985979b7-d563-86a0-47a1-f3babb49d205/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc3d834d51bb469485f350d5c225fbc4, entries=3, sequenceid=168, filesize=5.3 K 2024-12-04T04:29:14,140 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 593ms, sequenceid=168, compaction requested=false 2024-12-04T04:29:14,145 INFO [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T04:29:14,145 DEBUG [M:0;67abd3a0ee76:33125 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733286553546Disabling compacts and flushes for region at 1733286553546Disabling writes for close at 1733286553547 (+1 ms)Obtaining lock to block concurrent updates at 1733286553547Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733286553547Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733286553547Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733286553548 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733286553548Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733286553565 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733286553565Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733286553582 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733286553607 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733286553607Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733286554037 (+430 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733286554060 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733286554060Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@546669db: reopening flushed file at 1733286554107 (+47 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@738046ff: reopening flushed file at 1733286554118 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cac0bb4: reopening flushed file at 1733286554130 (+12 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 593ms, sequenceid=168, compaction requested=false at 1733286554140 (+10 ms)Writing region close event to WAL at 1733286554145 (+5 ms)Closed at 1733286554145 2024-12-04T04:29:14,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38591 is added to blk_1073741830_1006 (size=71349) 2024-12-04T04:29:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43031 is added to blk_1073741830_1006 (size=71349) 2024-12-04T04:29:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40387 is added to blk_1073741830_1006 (size=71349) 2024-12-04T04:29:14,172 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T04:29:14,172 INFO [M:0;67abd3a0ee76:33125 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T04:29:14,172 INFO [M:0;67abd3a0ee76:33125 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33125 2024-12-04T04:29:14,173 INFO [M:0;67abd3a0ee76:33125 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T04:29:14,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:14,304 INFO [M:0;67abd3a0ee76:33125 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T04:29:14,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33125-0x1019e9915430000, quorum=127.0.0.1:50801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T04:29:14,311 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286529066 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286529066 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733286528160/wal.1733286529066 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T04:29:14,312 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733286518050/wal.1733286518179 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:14,313 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733286546793/wal.1733286547005 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:14,318 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286539308 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286539308 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733286529764/wal.1733286539308 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T04:29:14,319 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733286510731/wal.1733286511101 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:14,323 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286541522 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286541522 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733286540488/wal.1733286541522 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T04:29:14,324 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733286541883/wal.1733286541987 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T04:29:14,326 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal.1733286510427 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal.1733286510427 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733286509454/wal.1733286510427 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T04:29:14,330 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal.1733286508762 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal.1733286508762 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733286507593/wal.1733286508762 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1523800604_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T04:29:14,344 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43206bef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:29:14,347 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@228ffa29{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T04:29:14,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T04:29:14,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f079a76{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T04:29:14,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da22a2e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,STOPPED} 2024-12-04T04:29:14,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T04:29:14,356 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T04:29:14,360 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T04:29:14,360 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-90967970-172.17.0.3-1733286495978 (Datanode Uuid 9dc7fb19-548d-43b9-8e6a-9d510388cda0) service to localhost/127.0.0.1:35021 2024-12-04T04:29:14,363 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data5/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,363 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data6/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,364 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T04:29:14,381 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4546bb60{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:29:14,382 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c7d32f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T04:29:14,382 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T04:29:14,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ebbf344{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T04:29:14,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b5fc47c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,STOPPED} 2024-12-04T04:29:14,388 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T04:29:14,388 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T04:29:14,388 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T04:29:14,388 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-90967970-172.17.0.3-1733286495978 (Datanode Uuid 16fde2f2-0eb2-4c6b-a3e1-474017a51da7) service to localhost/127.0.0.1:35021 2024-12-04T04:29:14,389 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data3/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,389 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T04:29:14,389 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data4/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,401 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@36632d60{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T04:29:14,401 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@751d2fa4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T04:29:14,401 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T04:29:14,402 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@433df981{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T04:29:14,402 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f76f489{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,STOPPED} 2024-12-04T04:29:14,404 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T04:29:14,404 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T04:29:14,404 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T04:29:14,404 WARN [BP-90967970-172.17.0.3-1733286495978 heartbeating to localhost/127.0.0.1:35021 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-90967970-172.17.0.3-1733286495978 (Datanode Uuid 426f4c1d-e284-418e-b4c4-90e5c05285b1) service to localhost/127.0.0.1:35021 2024-12-04T04:29:14,406 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data1/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,406 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/cluster_e69a1f06-d056-687d-ed4f-316525dd0f41/data/data2/current/BP-90967970-172.17.0.3-1733286495978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T04:29:14,406 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T04:29:14,416 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T04:29:14,421 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T04:29:14,421 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T04:29:14,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T04:29:14,421 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c8979d68-8bf6-70ab-67b5-e102b75a7d5a/hadoop.log.dir/,STOPPED} 2024-12-04T04:29:14,434 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T04:29:14,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down