2024-11-20 04:29:25,472 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 04:29:25,487 main DEBUG Took 0.012566 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 04:29:25,487 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 04:29:25,488 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 04:29:25,489 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 04:29:25,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,500 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 04:29:25,511 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,513 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,514 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,515 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,515 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,516 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,517 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,518 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,518 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,519 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,520 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,520 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 04:29:25,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,521 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 04:29:25,523 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 04:29:25,524 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 04:29:25,526 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 04:29:25,526 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 04:29:25,528 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 04:29:25,528 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 04:29:25,536 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 04:29:25,539 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 04:29:25,541 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 04:29:25,541 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 04:29:25,541 main DEBUG createAppenders(={Console}) 2024-11-20 04:29:25,542 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-20 04:29:25,542 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 04:29:25,542 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-20 04:29:25,543 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 04:29:25,543 main DEBUG OutputStream closed 2024-11-20 04:29:25,543 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 04:29:25,544 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 04:29:25,544 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-20 04:29:25,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 04:29:25,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 04:29:25,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 04:29:25,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 04:29:25,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 04:29:25,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 04:29:25,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 04:29:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 04:29:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 04:29:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 04:29:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 04:29:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 04:29:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 04:29:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 04:29:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 04:29:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 04:29:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 04:29:25,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 04:29:25,644 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 04:29:25,645 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-20 04:29:25,645 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 04:29:25,645 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-20T04:29:25,910 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc 2024-11-20 04:29:25,914 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 04:29:25,915 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T04:29:25,924 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-20T04:29:25,931 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-20T04:29:25,955 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T04:29:26,002 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T04:29:26,002 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T04:29:26,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T04:29:26,029 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6, deleteOnExit=true 2024-11-20T04:29:26,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T04:29:26,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/test.cache.data in system properties and HBase conf 2024-11-20T04:29:26,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T04:29:26,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir in system properties and HBase conf 2024-11-20T04:29:26,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T04:29:26,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T04:29:26,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T04:29:26,113 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T04:29:26,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T04:29:26,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T04:29:26,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T04:29:26,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T04:29:26,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T04:29:26,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T04:29:26,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T04:29:26,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T04:29:26,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T04:29:26,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T04:29:26,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/nfs.dump.dir in system properties and HBase conf 2024-11-20T04:29:26,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/java.io.tmpdir in system properties and HBase conf 2024-11-20T04:29:26,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T04:29:26,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T04:29:26,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T04:29:27,190 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T04:29:27,283 INFO [Time-limited test {}] log.Log(170): Logging initialized @2501ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T04:29:27,370 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T04:29:27,450 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T04:29:27,494 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T04:29:27,494 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T04:29:27,496 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T04:29:27,518 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T04:29:27,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,AVAILABLE} 2024-11-20T04:29:27,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T04:29:27,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/java.io.tmpdir/jetty-localhost-37765-hadoop-hdfs-3_4_1-tests_jar-_-any-345779713827129444/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T04:29:27,749 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:37765} 2024-11-20T04:29:27,750 INFO [Time-limited test {}] server.Server(415): Started @2968ms 2024-11-20T04:29:28,230 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T04:29:28,239 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T04:29:28,242 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T04:29:28,242 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T04:29:28,242 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T04:29:28,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55114173{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,AVAILABLE} 2024-11-20T04:29:28,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T04:29:28,421 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4da465b6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/java.io.tmpdir/jetty-localhost-36447-hadoop-hdfs-3_4_1-tests_jar-_-any-7969822323499915804/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:29:28,422 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c26fca4{HTTP/1.1, (http/1.1)}{localhost:36447} 2024-11-20T04:29:28,422 INFO [Time-limited test {}] server.Server(415): Started @3640ms 2024-11-20T04:29:28,489 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T04:29:28,625 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T04:29:28,634 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T04:29:28,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T04:29:28,638 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T04:29:28,638 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T04:29:28,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f53d973{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,AVAILABLE} 2024-11-20T04:29:28,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T04:29:28,806 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26760fc8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/java.io.tmpdir/jetty-localhost-43669-hadoop-hdfs-3_4_1-tests_jar-_-any-15062596542834810177/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:29:28,807 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26b9ecc1{HTTP/1.1, (http/1.1)}{localhost:43669} 2024-11-20T04:29:28,808 INFO [Time-limited test {}] server.Server(415): Started @4026ms 2024-11-20T04:29:28,811 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T04:29:28,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T04:29:28,881 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T04:29:28,886 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T04:29:28,887 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T04:29:28,887 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T04:29:28,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7701b046{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,AVAILABLE} 2024-11-20T04:29:28,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T04:29:29,027 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data1/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,027 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data4/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,028 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data2/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,030 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data3/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,055 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f8545c0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/java.io.tmpdir/jetty-localhost-44321-hadoop-hdfs-3_4_1-tests_jar-_-any-72608954568433353/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:29:29,057 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@664ea373{HTTP/1.1, (http/1.1)}{localhost:44321} 2024-11-20T04:29:29,057 INFO [Time-limited test {}] server.Server(415): Started @4275ms 2024-11-20T04:29:29,061 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T04:29:29,129 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T04:29:29,138 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T04:29:29,237 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a90a46e7c078af1 with lease ID 0x1d5731fe24d9fa5d: Processing first storage report for DS-2bf7ec94-06e8-4123-a913-acf570aee653 from datanode DatanodeRegistration(127.0.0.1:43305, datanodeUuid=61f47230-ad4a-4c9f-a628-a45306f25a26, infoPort=41839, infoSecurePort=0, ipcPort=40331, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,239 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a90a46e7c078af1 with lease ID 0x1d5731fe24d9fa5d: from storage DS-2bf7ec94-06e8-4123-a913-acf570aee653 node DatanodeRegistration(127.0.0.1:43305, datanodeUuid=61f47230-ad4a-4c9f-a628-a45306f25a26, infoPort=41839, infoSecurePort=0, ipcPort=40331, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,239 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd28b236c1de3a244 with lease ID 0x1d5731fe24d9fa5c: Processing first storage report for DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c from datanode DatanodeRegistration(127.0.0.1:46091, datanodeUuid=50d4a516-a147-488b-84ab-55496d172715, infoPort=40049, infoSecurePort=0, ipcPort=37089, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd28b236c1de3a244 with lease ID 0x1d5731fe24d9fa5c: from storage DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c node DatanodeRegistration(127.0.0.1:46091, datanodeUuid=50d4a516-a147-488b-84ab-55496d172715, infoPort=40049, infoSecurePort=0, ipcPort=37089, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a90a46e7c078af1 with lease ID 0x1d5731fe24d9fa5d: Processing first storage report for DS-303b56b4-8c1b-4579-80f8-08f9cf26bb8f from datanode DatanodeRegistration(127.0.0.1:43305, datanodeUuid=61f47230-ad4a-4c9f-a628-a45306f25a26, infoPort=41839, infoSecurePort=0, ipcPort=40331, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a90a46e7c078af1 with lease ID 0x1d5731fe24d9fa5d: from storage DS-303b56b4-8c1b-4579-80f8-08f9cf26bb8f node DatanodeRegistration(127.0.0.1:43305, datanodeUuid=61f47230-ad4a-4c9f-a628-a45306f25a26, infoPort=41839, infoSecurePort=0, ipcPort=40331, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd28b236c1de3a244 with lease ID 0x1d5731fe24d9fa5c: Processing first storage report for DS-950d46cd-1a38-4a9a-9bab-620881c2a422 from datanode DatanodeRegistration(127.0.0.1:46091, datanodeUuid=50d4a516-a147-488b-84ab-55496d172715, infoPort=40049, infoSecurePort=0, ipcPort=37089, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,240 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd28b236c1de3a244 with lease ID 0x1d5731fe24d9fa5c: from storage DS-950d46cd-1a38-4a9a-9bab-620881c2a422 node DatanodeRegistration(127.0.0.1:46091, datanodeUuid=50d4a516-a147-488b-84ab-55496d172715, infoPort=40049, infoSecurePort=0, ipcPort=37089, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,334 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data5/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,335 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data6/current/BP-2101640850-172.17.0.2-1732076966903/current, will proceed with Du for space computation calculation, 2024-11-20T04:29:29,379 WARN [Thread-126 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T04:29:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f46b223c9d0111d with lease ID 0x1d5731fe24d9fa5e: Processing first storage report for DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b from datanode DatanodeRegistration(127.0.0.1:45985, datanodeUuid=6d34e9dd-5552-4279-8dab-2c7dce9ec7bf, infoPort=40607, infoSecurePort=0, ipcPort=36549, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f46b223c9d0111d with lease ID 0x1d5731fe24d9fa5e: from storage DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b node DatanodeRegistration(127.0.0.1:45985, datanodeUuid=6d34e9dd-5552-4279-8dab-2c7dce9ec7bf, infoPort=40607, infoSecurePort=0, ipcPort=36549, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1f46b223c9d0111d with lease ID 0x1d5731fe24d9fa5e: Processing first storage report for DS-ad96a626-ae4d-4b4b-8be8-413007dc84c0 from datanode DatanodeRegistration(127.0.0.1:45985, datanodeUuid=6d34e9dd-5552-4279-8dab-2c7dce9ec7bf, infoPort=40607, infoSecurePort=0, ipcPort=36549, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903) 2024-11-20T04:29:29,388 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1f46b223c9d0111d with lease ID 0x1d5731fe24d9fa5e: from storage DS-ad96a626-ae4d-4b4b-8be8-413007dc84c0 node DatanodeRegistration(127.0.0.1:45985, datanodeUuid=6d34e9dd-5552-4279-8dab-2c7dce9ec7bf, infoPort=40607, infoSecurePort=0, ipcPort=36549, storageInfo=lv=-57;cid=testClusterID;nsid=708563700;c=1732076966903), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T04:29:29,603 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc 2024-11-20T04:29:29,704 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/zookeeper_0, clientPort=60520, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T04:29:29,714 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60520 2024-11-20T04:29:29,741 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:29,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741825_1001 (size=7) 2024-11-20T04:29:29,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741825_1001 (size=7) 2024-11-20T04:29:29,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741825_1001 (size=7) 2024-11-20T04:29:30,383 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e with version=8 2024-11-20T04:29:30,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/hbase-staging 2024-11-20T04:29:30,751 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2a32e16c274:0 server-side Connection retries=45 2024-11-20T04:29:30,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:30,764 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:30,770 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T04:29:30,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:30,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T04:29:30,935 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T04:29:31,000 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T04:29:31,012 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T04:29:31,017 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T04:29:31,049 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 23671 (auto-detected) 2024-11-20T04:29:31,050 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T04:29:31,075 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37809 2024-11-20T04:29:31,101 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37809 connecting to ZooKeeper ensemble=127.0.0.1:60520 2024-11-20T04:29:31,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378090x0, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T04:29:31,139 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37809-0x10133a3095f0000 connected 2024-11-20T04:29:31,175 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,190 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:29:31,194 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e, hbase.cluster.distributed=false 2024-11-20T04:29:31,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T04:29:31,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37809 2024-11-20T04:29:31,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37809 2024-11-20T04:29:31,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37809 2024-11-20T04:29:31,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37809 2024-11-20T04:29:31,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37809 2024-11-20T04:29:31,354 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2a32e16c274:0 server-side Connection retries=45 2024-11-20T04:29:31,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,356 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T04:29:31,357 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,357 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T04:29:31,360 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T04:29:31,362 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T04:29:31,364 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36039 2024-11-20T04:29:31,366 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36039 connecting to ZooKeeper ensemble=127.0.0.1:60520 2024-11-20T04:29:31,368 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360390x0, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T04:29:31,378 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36039-0x10133a3095f0001 connected 2024-11-20T04:29:31,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:29:31,383 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T04:29:31,391 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T04:29:31,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T04:29:31,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T04:29:31,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36039 2024-11-20T04:29:31,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36039 2024-11-20T04:29:31,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36039 2024-11-20T04:29:31,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36039 2024-11-20T04:29:31,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36039 2024-11-20T04:29:31,422 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2a32e16c274:0 server-side Connection retries=45 2024-11-20T04:29:31,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T04:29:31,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T04:29:31,424 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T04:29:31,424 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T04:29:31,426 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38897 2024-11-20T04:29:31,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38897 connecting to ZooKeeper ensemble=127.0.0.1:60520 2024-11-20T04:29:31,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388970x0, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T04:29:31,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388970x0, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:29:31,441 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T04:29:31,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38897-0x10133a3095f0002 connected 2024-11-20T04:29:31,454 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T04:29:31,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T04:29:31,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T04:29:31,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-20T04:29:31,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38897 2024-11-20T04:29:31,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38897 2024-11-20T04:29:31,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-20T04:29:31,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-20T04:29:31,500 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2a32e16c274:0 server-side Connection retries=45 2024-11-20T04:29:31,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,501 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T04:29:31,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T04:29:31,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T04:29:31,502 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T04:29:31,503 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T04:29:31,505 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39781 2024-11-20T04:29:31,507 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39781 connecting to ZooKeeper ensemble=127.0.0.1:60520 2024-11-20T04:29:31,510 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,513 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,522 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397810x0, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T04:29:31,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397810x0, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:29:31,524 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T04:29:31,525 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39781-0x10133a3095f0003 connected 2024-11-20T04:29:31,540 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T04:29:31,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T04:29:31,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T04:29:31,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39781 2024-11-20T04:29:31,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39781 2024-11-20T04:29:31,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39781 2024-11-20T04:29:31,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39781 2024-11-20T04:29:31,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39781 2024-11-20T04:29:31,585 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2a32e16c274:37809 2024-11-20T04:29:31,587 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2a32e16c274,37809,1732076970521 2024-11-20T04:29:31,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,595 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,602 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2a32e16c274,37809,1732076970521 2024-11-20T04:29:31,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T04:29:31,625 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T04:29:31,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T04:29:31,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,626 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,631 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T04:29:31,632 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2a32e16c274,37809,1732076970521 from backup master directory 2024-11-20T04:29:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2a32e16c274,37809,1732076970521 2024-11-20T04:29:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,636 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T04:29:31,638 WARN [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T04:29:31,638 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2a32e16c274,37809,1732076970521 2024-11-20T04:29:31,641 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T04:29:31,644 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T04:29:31,720 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/hbase.id] with ID: 80f02adf-6f24-435e-89f5-b374cfcfd2bf 2024-11-20T04:29:31,720 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/.tmp/hbase.id 2024-11-20T04:29:31,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741826_1002 (size=42) 2024-11-20T04:29:31,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741826_1002 (size=42) 2024-11-20T04:29:31,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741826_1002 (size=42) 2024-11-20T04:29:31,747 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/.tmp/hbase.id]:[hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/hbase.id] 2024-11-20T04:29:31,810 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T04:29:31,817 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T04:29:31,840 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-20T04:29:31,846 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741827_1003 (size=196) 2024-11-20T04:29:31,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741827_1003 (size=196) 2024-11-20T04:29:31,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741827_1003 (size=196) 2024-11-20T04:29:31,892 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T04:29:31,894 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T04:29:31,902 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:31,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741828_1004 (size=1189) 2024-11-20T04:29:31,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741828_1004 (size=1189) 2024-11-20T04:29:31,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741828_1004 (size=1189) 2024-11-20T04:29:31,958 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store 2024-11-20T04:29:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741829_1005 (size=34) 2024-11-20T04:29:31,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741829_1005 (size=34) 2024-11-20T04:29:31,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741829_1005 (size=34) 2024-11-20T04:29:31,990 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T04:29:31,993 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:31,994 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T04:29:31,994 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:29:31,994 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:29:31,996 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T04:29:31,996 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:29:31,996 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:29:31,997 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732076971994Disabling compacts and flushes for region at 1732076971994Disabling writes for close at 1732076971996 (+2 ms)Writing region close event to WAL at 1732076971996Closed at 1732076971996 2024-11-20T04:29:32,000 WARN [master/c2a32e16c274:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/.initializing 2024-11-20T04:29:32,000 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/WALs/c2a32e16c274,37809,1732076970521 2024-11-20T04:29:32,009 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:32,026 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2a32e16c274%2C37809%2C1732076970521, suffix=, logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/WALs/c2a32e16c274,37809,1732076970521, archiveDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/oldWALs, maxLogs=10 2024-11-20T04:29:32,068 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/WALs/c2a32e16c274,37809,1732076970521/c2a32e16c274%2C37809%2C1732076970521.1732076972031, exclude list is [], retry=0 2024-11-20T04:29:32,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:32,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:32,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:32,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:32,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T04:29:32,156 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/WALs/c2a32e16c274,37809,1732076970521/c2a32e16c274%2C37809%2C1732076970521.1732076972031 2024-11-20T04:29:32,158 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:32,159 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:32,160 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:32,164 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,165 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T04:29:32,263 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:32,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T04:29:32,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:32,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T04:29:32,282 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:32,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T04:29:32,287 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:32,289 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,293 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,294 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,300 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,301 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,305 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T04:29:32,311 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T04:29:32,316 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:32,318 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68625872, jitterRate=0.022605180740356445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T04:29:32,326 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732076972184Initializing all the Stores at 1732076972186 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076972187 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076972188 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076972188Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076972188Cleaning up temporary data from old regions at 1732076972301 (+113 ms)Region opened successfully at 1732076972325 (+24 ms) 2024-11-20T04:29:32,327 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T04:29:32,368 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7098193a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2a32e16c274/172.17.0.2:0 2024-11-20T04:29:32,414 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T04:29:32,429 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T04:29:32,429 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T04:29:32,433 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T04:29:32,435 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T04:29:32,440 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T04:29:32,440 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T04:29:32,472 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T04:29:32,482 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T04:29:32,484 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T04:29:32,488 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T04:29:32,489 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T04:29:32,494 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T04:29:32,507 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T04:29:32,515 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T04:29:32,518 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T04:29:32,520 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T04:29:32,524 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T04:29:32,547 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T04:29:32,549 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T04:29:32,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T04:29:32,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T04:29:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T04:29:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,555 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T04:29:32,555 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,559 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2a32e16c274,37809,1732076970521, sessionid=0x10133a3095f0000, setting cluster-up flag (Was=false) 2024-11-20T04:29:32,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,577 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,585 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T04:29:32,587 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2a32e16c274,37809,1732076970521 2024-11-20T04:29:32,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,593 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:32,602 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T04:29:32,603 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2a32e16c274,37809,1732076970521 2024-11-20T04:29:32,610 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T04:29:32,669 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(746): ClusterId : 80f02adf-6f24-435e-89f5-b374cfcfd2bf 2024-11-20T04:29:32,669 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(746): ClusterId : 80f02adf-6f24-435e-89f5-b374cfcfd2bf 2024-11-20T04:29:32,669 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(746): ClusterId : 80f02adf-6f24-435e-89f5-b374cfcfd2bf 2024-11-20T04:29:32,673 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T04:29:32,673 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T04:29:32,673 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T04:29:32,678 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T04:29:32,678 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T04:29:32,679 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T04:29:32,679 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T04:29:32,679 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T04:29:32,679 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T04:29:32,683 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T04:29:32,683 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T04:29:32,683 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T04:29:32,684 DEBUG [RS:2;c2a32e16c274:39781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ee7c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2a32e16c274/172.17.0.2:0 2024-11-20T04:29:32,684 DEBUG [RS:1;c2a32e16c274:38897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abb16e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2a32e16c274/172.17.0.2:0 2024-11-20T04:29:32,684 DEBUG [RS:0;c2a32e16c274:36039 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42af3426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2a32e16c274/172.17.0.2:0 2024-11-20T04:29:32,697 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T04:29:32,704 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2a32e16c274:36039 2024-11-20T04:29:32,708 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T04:29:32,708 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T04:29:32,708 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T04:29:32,710 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T04:29:32,711 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2a32e16c274,37809,1732076970521 with port=36039, startcode=1732076971312 2024-11-20T04:29:32,717 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c2a32e16c274:39781 2024-11-20T04:29:32,717 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T04:29:32,717 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T04:29:32,718 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T04:29:32,719 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2a32e16c274,37809,1732076970521 with port=39781, startcode=1732076971499 2024-11-20T04:29:32,720 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T04:29:32,721 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c2a32e16c274:38897 2024-11-20T04:29:32,721 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T04:29:32,721 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T04:29:32,721 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T04:29:32,722 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2a32e16c274,37809,1732076970521 with port=38897, startcode=1732076971421 2024-11-20T04:29:32,727 DEBUG [RS:2;c2a32e16c274:39781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T04:29:32,727 DEBUG [RS:0;c2a32e16c274:36039 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T04:29:32,727 DEBUG [RS:1;c2a32e16c274:38897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T04:29:32,731 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2a32e16c274,37809,1732076970521 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T04:29:32,740 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2a32e16c274:0, corePoolSize=5, maxPoolSize=5 2024-11-20T04:29:32,740 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2a32e16c274:0, corePoolSize=5, maxPoolSize=5 2024-11-20T04:29:32,740 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2a32e16c274:0, corePoolSize=5, maxPoolSize=5 2024-11-20T04:29:32,740 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2a32e16c274:0, corePoolSize=5, maxPoolSize=5 2024-11-20T04:29:32,741 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2a32e16c274:0, corePoolSize=10, maxPoolSize=10 2024-11-20T04:29:32,741 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,741 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2a32e16c274:0, corePoolSize=2, maxPoolSize=2 2024-11-20T04:29:32,741 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,746 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732077002746 2024-11-20T04:29:32,747 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T04:29:32,749 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T04:29:32,753 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T04:29:32,753 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T04:29:32,754 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T04:29:32,754 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T04:29:32,754 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,759 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T04:29:32,761 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T04:29:32,762 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T04:29:32,764 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T04:29:32,764 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T04:29:32,765 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T04:29:32,765 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T04:29:32,767 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.large.0-1732076972766,5,FailOnTimeoutGroup] 2024-11-20T04:29:32,767 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.small.0-1732076972767,5,FailOnTimeoutGroup] 2024-11-20T04:29:32,768 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,768 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T04:29:32,769 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,770 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,774 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37481, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T04:29:32,775 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,775 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46117, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T04:29:32,774 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54041, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T04:29:32,776 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T04:29:32,783 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(517): Registering regionserver=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741831_1007 (size=1321) 2024-11-20T04:29:32,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741831_1007 (size=1321) 2024-11-20T04:29:32,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741831_1007 (size=1321) 2024-11-20T04:29:32,827 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T04:29:32,828 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e 2024-11-20T04:29:32,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(517): Registering regionserver=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,839 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e 2024-11-20T04:29:32,839 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42657 2024-11-20T04:29:32,839 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T04:29:32,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37809 {}] master.ServerManager(517): Registering regionserver=c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,843 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e 2024-11-20T04:29:32,843 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42657 2024-11-20T04:29:32,843 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T04:29:32,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T04:29:32,846 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e 2024-11-20T04:29:32,847 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42657 2024-11-20T04:29:32,847 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T04:29:32,850 DEBUG [RS:0;c2a32e16c274:36039 {}] zookeeper.ZKUtil(111): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2a32e16c274,36039,1732076971312] 2024-11-20T04:29:32,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2a32e16c274,38897,1732076971421] 2024-11-20T04:29:32,850 WARN [RS:0;c2a32e16c274:36039 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T04:29:32,851 INFO [RS:0;c2a32e16c274:36039 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:32,851 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741832_1008 (size=32) 2024-11-20T04:29:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741832_1008 (size=32) 2024-11-20T04:29:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741832_1008 (size=32) 2024-11-20T04:29:32,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T04:29:32,855 DEBUG [RS:1;c2a32e16c274:38897 {}] zookeeper.ZKUtil(111): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,856 WARN [RS:1;c2a32e16c274:38897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T04:29:32,856 INFO [RS:1;c2a32e16c274:38897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:32,856 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,856 DEBUG [RS:2;c2a32e16c274:39781 {}] zookeeper.ZKUtil(111): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,856 WARN [RS:2;c2a32e16c274:39781 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T04:29:32,857 INFO [RS:2;c2a32e16c274:39781 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:32,857 DEBUG [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:32,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2a32e16c274,39781,1732076971499] 2024-11-20T04:29:32,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T04:29:32,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T04:29:32,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:32,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T04:29:32,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T04:29:32,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:32,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T04:29:32,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T04:29:32,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:32,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T04:29:32,882 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T04:29:32,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:32,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:32,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T04:29:32,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740 2024-11-20T04:29:32,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740 2024-11-20T04:29:32,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T04:29:32,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T04:29:32,890 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T04:29:32,892 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T04:29:32,892 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T04:29:32,892 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T04:29:32,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T04:29:32,897 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:32,898 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70944109, jitterRate=0.057149603962898254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T04:29:32,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732076972858Initializing all the Stores at 1732076972859 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076972859Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076972861 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076972861Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076972861Cleaning up temporary data from old regions at 1732076972889 (+28 ms)Region opened successfully at 1732076972901 (+12 ms) 2024-11-20T04:29:32,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T04:29:32,902 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T04:29:32,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T04:29:32,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T04:29:32,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T04:29:32,904 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T04:29:32,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732076972902Disabling compacts and flushes for region at 1732076972902Disabling writes for close at 1732076972902Writing region close event to WAL at 1732076972903 (+1 ms)Closed at 1732076972904 (+1 ms) 2024-11-20T04:29:32,908 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T04:29:32,908 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T04:29:32,908 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T04:29:32,908 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T04:29:32,909 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T04:29:32,915 INFO [RS:2;c2a32e16c274:39781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T04:29:32,915 INFO [RS:0;c2a32e16c274:36039 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T04:29:32,915 INFO [RS:1;c2a32e16c274:38897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T04:29:32,915 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,915 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,915 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,916 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T04:29:32,916 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T04:29:32,917 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T04:29:32,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T04:29:32,924 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T04:29:32,924 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T04:29:32,924 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T04:29:32,926 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,926 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,926 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,926 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,926 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0, corePoolSize=2, maxPoolSize=2 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0, corePoolSize=2, maxPoolSize=2 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0, corePoolSize=2, maxPoolSize=2 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,927 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:2;c2a32e16c274:39781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2a32e16c274:0, corePoolSize=1, maxPoolSize=1 2024-11-20T04:29:32,928 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,928 DEBUG [RS:1;c2a32e16c274:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,928 DEBUG [RS:0;c2a32e16c274:36039 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2a32e16c274:0, corePoolSize=3, maxPoolSize=3 2024-11-20T04:29:32,929 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T04:29:32,930 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,930 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,931 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,931 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,931 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,931 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,39781,1732076971499-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T04:29:32,935 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T04:29:32,941 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,941 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,941 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,941 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,941 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,36039,1732076971312-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T04:29:32,942 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,38897,1732076971421-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T04:29:32,964 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T04:29:32,965 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T04:29:32,966 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,39781,1732076971499-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,966 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,38897,1732076971421-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,966 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,966 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.Replication(171): c2a32e16c274,39781,1732076971499 started 2024-11-20T04:29:32,966 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,966 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.Replication(171): c2a32e16c274,38897,1732076971421 started 2024-11-20T04:29:32,969 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T04:29:32,969 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,36039,1732076971312-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,970 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,970 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.Replication(171): c2a32e16c274,36039,1732076971312 started 2024-11-20T04:29:32,984 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,984 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,984 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1482): Serving as c2a32e16c274,39781,1732076971499, RpcServer on c2a32e16c274/172.17.0.2:39781, sessionid=0x10133a3095f0003 2024-11-20T04:29:32,985 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1482): Serving as c2a32e16c274,38897,1732076971421, RpcServer on c2a32e16c274/172.17.0.2:38897, sessionid=0x10133a3095f0002 2024-11-20T04:29:32,985 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T04:29:32,985 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T04:29:32,985 DEBUG [RS:2;c2a32e16c274:39781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,985 DEBUG [RS:1;c2a32e16c274:38897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,986 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,39781,1732076971499' 2024-11-20T04:29:32,986 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,38897,1732076971421' 2024-11-20T04:29:32,986 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T04:29:32,986 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T04:29:32,987 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T04:29:32,987 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T04:29:32,987 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T04:29:32,987 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T04:29:32,987 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T04:29:32,987 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T04:29:32,988 DEBUG [RS:1;c2a32e16c274:38897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2a32e16c274,38897,1732076971421 2024-11-20T04:29:32,988 DEBUG [RS:2;c2a32e16c274:39781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2a32e16c274,39781,1732076971499 2024-11-20T04:29:32,988 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,38897,1732076971421' 2024-11-20T04:29:32,988 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,39781,1732076971499' 2024-11-20T04:29:32,988 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T04:29:32,988 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T04:29:32,988 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T04:29:32,988 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T04:29:32,989 DEBUG [RS:1;c2a32e16c274:38897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T04:29:32,989 INFO [RS:1;c2a32e16c274:38897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T04:29:32,989 DEBUG [RS:2;c2a32e16c274:39781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T04:29:32,989 INFO [RS:1;c2a32e16c274:38897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T04:29:32,989 INFO [RS:2;c2a32e16c274:39781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T04:29:32,989 INFO [RS:2;c2a32e16c274:39781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T04:29:32,991 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:32,992 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1482): Serving as c2a32e16c274,36039,1732076971312, RpcServer on c2a32e16c274/172.17.0.2:36039, sessionid=0x10133a3095f0001 2024-11-20T04:29:32,992 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T04:29:32,992 DEBUG [RS:0;c2a32e16c274:36039 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,992 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,36039,1732076971312' 2024-11-20T04:29:32,992 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T04:29:32,993 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T04:29:32,993 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T04:29:32,993 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T04:29:32,993 DEBUG [RS:0;c2a32e16c274:36039 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2a32e16c274,36039,1732076971312 2024-11-20T04:29:32,993 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2a32e16c274,36039,1732076971312' 2024-11-20T04:29:32,994 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T04:29:32,994 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T04:29:32,995 DEBUG [RS:0;c2a32e16c274:36039 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T04:29:32,995 INFO [RS:0;c2a32e16c274:36039 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T04:29:32,995 INFO [RS:0;c2a32e16c274:36039 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T04:29:33,086 WARN [c2a32e16c274:37809 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T04:29:33,095 INFO [RS:1;c2a32e16c274:38897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:33,095 INFO [RS:0;c2a32e16c274:36039 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:33,096 INFO [RS:2;c2a32e16c274:39781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:33,115 INFO [RS:0;c2a32e16c274:36039 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2a32e16c274%2C36039%2C1732076971312, suffix=, logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312, archiveDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs, maxLogs=32 2024-11-20T04:29:33,116 INFO [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2a32e16c274%2C38897%2C1732076971421, suffix=, logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421, archiveDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs, maxLogs=32 2024-11-20T04:29:33,117 INFO [RS:2;c2a32e16c274:39781 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2a32e16c274%2C39781%2C1732076971499, suffix=, logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,39781,1732076971499, archiveDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs, maxLogs=32 2024-11-20T04:29:33,139 DEBUG [RS:0;c2a32e16c274:36039 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312/c2a32e16c274%2C36039%2C1732076971312.1732076973120, exclude list is [], retry=0 2024-11-20T04:29:33,144 DEBUG [RS:2;c2a32e16c274:39781 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,39781,1732076971499/c2a32e16c274%2C39781%2C1732076971499.1732076973120, exclude list is [], retry=0 2024-11-20T04:29:33,144 DEBUG [RS:1;c2a32e16c274:38897 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421/c2a32e16c274%2C38897%2C1732076971421.1732076973120, exclude list is [], retry=0 2024-11-20T04:29:33,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:33,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:33,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:33,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:33,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:33,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:33,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:33,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:33,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:33,168 INFO [RS:0;c2a32e16c274:36039 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312/c2a32e16c274%2C36039%2C1732076971312.1732076973120 2024-11-20T04:29:33,169 DEBUG [RS:0;c2a32e16c274:36039 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:33,170 INFO [RS:2;c2a32e16c274:39781 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,39781,1732076971499/c2a32e16c274%2C39781%2C1732076971499.1732076973120 2024-11-20T04:29:33,172 DEBUG [RS:2;c2a32e16c274:39781 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:33,173 INFO [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421/c2a32e16c274%2C38897%2C1732076971421.1732076973120 2024-11-20T04:29:33,173 DEBUG [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:33,339 DEBUG [c2a32e16c274:37809 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T04:29:33,350 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(204): Hosts are {c2a32e16c274=0} racks are {/default-rack=0} 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T04:29:33,358 INFO [c2a32e16c274:37809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T04:29:33,358 INFO [c2a32e16c274:37809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T04:29:33,358 INFO [c2a32e16c274:37809 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T04:29:33,358 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T04:29:33,369 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:33,375 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2a32e16c274,38897,1732076971421, state=OPENING 2024-11-20T04:29:33,382 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T04:29:33,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:33,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:33,384 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:33,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:29:33,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,388 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T04:29:33,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2a32e16c274,38897,1732076971421}] 2024-11-20T04:29:33,569 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T04:29:33,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44053, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T04:29:33,584 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T04:29:33,585 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:33,586 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T04:29:33,589 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2a32e16c274%2C38897%2C1732076971421.meta, suffix=.meta, logDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421, archiveDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs, maxLogs=32 2024-11-20T04:29:33,609 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421/c2a32e16c274%2C38897%2C1732076971421.meta.1732076973591.meta, exclude list is [], retry=0 2024-11-20T04:29:33,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:33,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:33,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:33,617 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,38897,1732076971421/c2a32e16c274%2C38897%2C1732076971421.meta.1732076973591.meta 2024-11-20T04:29:33,617 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:33,618 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:33,619 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T04:29:33,622 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T04:29:33,627 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T04:29:33,632 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T04:29:33,632 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:33,633 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T04:29:33,633 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T04:29:33,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T04:29:33,637 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T04:29:33,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:33,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:33,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T04:29:33,640 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T04:29:33,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:33,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:33,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T04:29:33,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T04:29:33,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:33,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:33,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T04:29:33,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T04:29:33,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:33,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T04:29:33,646 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T04:29:33,647 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740 2024-11-20T04:29:33,650 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740 2024-11-20T04:29:33,654 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T04:29:33,654 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T04:29:33,655 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T04:29:33,658 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T04:29:33,660 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72364680, jitterRate=0.07831776142120361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T04:29:33,660 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T04:29:33,662 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732076973633Writing region info on filesystem at 1732076973633Initializing all the Stores at 1732076973635 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076973635Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076973636 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076973636Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732076973636Cleaning up temporary data from old regions at 1732076973654 (+18 ms)Running coprocessor post-open hooks at 1732076973660 (+6 ms)Region opened successfully at 1732076973662 (+2 ms) 2024-11-20T04:29:33,669 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732076973560 2024-11-20T04:29:33,681 DEBUG [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T04:29:33,682 INFO [RS_OPEN_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T04:29:33,683 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:33,685 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2a32e16c274,38897,1732076971421, state=OPEN 2024-11-20T04:29:33,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T04:29:33,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T04:29:33,688 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T04:29:33,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T04:29:33,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T04:29:33,689 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:33,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T04:29:33,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2a32e16c274,38897,1732076971421 in 300 msec 2024-11-20T04:29:33,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T04:29:33,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 780 msec 2024-11-20T04:29:33,702 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T04:29:33,702 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T04:29:33,727 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T04:29:33,729 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2a32e16c274,38897,1732076971421, seqNum=-1] 2024-11-20T04:29:33,758 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T04:29:33,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51169, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T04:29:33,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1410 sec 2024-11-20T04:29:33,784 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732076973784, completionTime=-1 2024-11-20T04:29:33,787 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T04:29:33,787 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T04:29:33,822 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T04:29:33,822 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732077033822 2024-11-20T04:29:33,822 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732077093822 2024-11-20T04:29:33,822 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-20T04:29:33,824 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T04:29:33,831 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,831 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,831 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,833 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2a32e16c274:37809, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,833 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,836 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,840 DEBUG [master/c2a32e16c274:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T04:29:33,892 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.253sec 2024-11-20T04:29:33,893 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T04:29:33,894 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T04:29:33,895 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T04:29:33,896 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T04:29:33,896 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T04:29:33,897 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T04:29:33,898 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T04:29:33,907 DEBUG [master/c2a32e16c274:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T04:29:33,908 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T04:29:33,909 INFO [master/c2a32e16c274:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2a32e16c274,37809,1732076970521-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:33,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7afd5a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T04:29:33,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2a32e16c274,37809,-1 for getting cluster id 2024-11-20T04:29:33,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T04:29:33,994 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80f02adf-6f24-435e-89f5-b374cfcfd2bf' 2024-11-20T04:29:33,996 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T04:29:33,997 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80f02adf-6f24-435e-89f5-b374cfcfd2bf" 2024-11-20T04:29:33,997 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d0745e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T04:29:33,997 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2a32e16c274,37809,-1] 2024-11-20T04:29:34,000 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T04:29:34,001 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:29:34,003 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T04:29:34,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad28ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T04:29:34,006 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T04:29:34,013 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2a32e16c274,38897,1732076971421, seqNum=-1] 2024-11-20T04:29:34,013 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T04:29:34,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T04:29:34,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2a32e16c274,37809,1732076970521 2024-11-20T04:29:34,039 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:42657/hbase 2024-11-20T04:29:34,058 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=7245 2024-11-20T04:29:34,082 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,086 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:34,091 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11663347, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-11663347, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:34,107 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11663347/hregion-11663347.1732076974092, exclude list is [], retry=0 2024-11-20T04:29:34,113 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:34,113 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:34,114 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:34,119 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11663347/hregion-11663347.1732076974092 2024-11-20T04:29:34,120 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:34,121 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8ea9036ad5b93db8385ce6c5d98c0ad1, NAME => 'testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:34,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741838_1014 (size=64) 2024-11-20T04:29:34,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741838_1014 (size=64) 2024-11-20T04:29:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741838_1014 (size=64) 2024-11-20T04:29:34,145 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:34,148 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,150 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ea9036ad5b93db8385ce6c5d98c0ad1 columnFamilyName a 2024-11-20T04:29:34,150 DEBUG [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:34,151 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] regionserver.HStore(327): Store=8ea9036ad5b93db8385ce6c5d98c0ad1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:34,152 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,154 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,154 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,158 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,162 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:34,163 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8ea9036ad5b93db8385ce6c5d98c0ad1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58909178, jitterRate=-0.12218484282493591}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:34,164 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8ea9036ad5b93db8385ce6c5d98c0ad1: Writing region info on filesystem at 1732076974146Initializing all the Stores at 1732076974147 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076974147Cleaning up temporary data from old regions at 1732076974154 (+7 ms)Region opened successfully at 1732076974164 (+10 ms) 2024-11-20T04:29:34,165 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8ea9036ad5b93db8385ce6c5d98c0ad1, disabling compactions & flushes 2024-11-20T04:29:34,165 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. 2024-11-20T04:29:34,165 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. 2024-11-20T04:29:34,165 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. after waiting 0 ms 2024-11-20T04:29:34,165 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. 2024-11-20T04:29:34,165 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. 2024-11-20T04:29:34,166 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8ea9036ad5b93db8385ce6c5d98c0ad1: Waiting for close lock at 1732076974165Disabling compacts and flushes for region at 1732076974165Disabling writes for close at 1732076974165Writing region close event to WAL at 1732076974165Closed at 1732076974165 2024-11-20T04:29:34,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741837_1013 (size=93) 2024-11-20T04:29:34,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741837_1013 (size=93) 2024-11-20T04:29:34,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741837_1013 (size=93) 2024-11-20T04:29:34,178 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:34,178 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11663347:(num 1732076974092) 2024-11-20T04:29:34,180 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T04:29:34,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741839_1015 (size=276) 2024-11-20T04:29:34,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741839_1015 (size=276) 2024-11-20T04:29:34,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741839_1015 (size=276) 2024-11-20T04:29:34,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T04:29:34,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741840_1016 (size=230) 2024-11-20T04:29:34,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741840_1016 (size=230) 2024-11-20T04:29:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741840_1016 (size=230) 2024-11-20T04:29:34,236 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1, size=276 (276bytes) 2024-11-20T04:29:34,237 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T04:29:34,237 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T04:29:34,237 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1 2024-11-20T04:29:34,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1 after 4ms 2024-11-20T04:29:34,249 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,250 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1 took 16ms 2024-11-20T04:29:34,254 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1 so closing down 2024-11-20T04:29:34,257 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T04:29:34,259 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:34,259 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T04:29:34,261 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:34,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741841_1017 (size=276) 2024-11-20T04:29:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741841_1017 (size=276) 2024-11-20T04:29:34,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741841_1017 (size=276) 2024-11-20T04:29:34,273 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:34,276 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002 2024-11-20T04:29:34,280 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-20T04:29:34,280 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1, journal: Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1, size=276 (276bytes) at 1732076974236Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1 so closing down at 1732076974254 (+18 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp at 1732076974259 (+5 ms)3 split writer threads finished at 1732076974261 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732076974273 (+12 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002 at 1732076974276 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732076974280 (+4 ms) 2024-11-20T04:29:34,297 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2, size=230 (230bytes) 2024-11-20T04:29:34,297 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2 2024-11-20T04:29:34,298 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2 after 1ms 2024-11-20T04:29:34,302 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,302 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2 took 5ms 2024-11-20T04:29:34,304 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2 so closing down 2024-11-20T04:29:34,305 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:34,307 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T04:29:34,309 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T04:29:34,309 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:34,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741842_1018 (size=230) 2024-11-20T04:29:34,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741842_1018 (size=230) 2024-11-20T04:29:34,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741842_1018 (size=230) 2024-11-20T04:29:34,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:34,324 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,326 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-20T04:29:34,328 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-20T04:29:34,328 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2, journal: Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2, size=230 (230bytes) at 1732076974297Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2 so closing down at 1732076974304 (+7 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002-wal-2.temp at 1732076974309 (+5 ms)3 split writer threads finished at 1732076974309Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732076974319 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732076974328 (+9 ms) 2024-11-20T04:29:34,329 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:34,331 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:34,346 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal.1732076974332, exclude list is [], retry=0 2024-11-20T04:29:34,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:34,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:34,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:34,354 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal.1732076974332 2024-11-20T04:29:34,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:34,354 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ea9036ad5b93db8385ce6c5d98c0ad1, NAME => 'testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:34,354 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:34,354 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,354 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,357 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,358 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ea9036ad5b93db8385ce6c5d98c0ad1 columnFamilyName a 2024-11-20T04:29:34,358 DEBUG [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:34,359 INFO [StoreOpener-8ea9036ad5b93db8385ce6c5d98c0ad1-1 {}] regionserver.HStore(327): Store=8ea9036ad5b93db8385ce6c5d98c0ad1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:34,360 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,364 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,366 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002 2024-11-20T04:29:34,370 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,378 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002 2024-11-20T04:29:34,382 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8ea9036ad5b93db8385ce6c5d98c0ad1 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T04:29:34,455 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/.tmp/a/dbc20ae3c3654f74a04ab89eb38c8fcb is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732076974178/Put/seqid=0 2024-11-20T04:29:34,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741844_1020 (size=5170) 2024-11-20T04:29:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741844_1020 (size=5170) 2024-11-20T04:29:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741844_1020 (size=5170) 2024-11-20T04:29:34,472 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/.tmp/a/dbc20ae3c3654f74a04ab89eb38c8fcb 2024-11-20T04:29:34,524 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/.tmp/a/dbc20ae3c3654f74a04ab89eb38c8fcb as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/a/dbc20ae3c3654f74a04ab89eb38c8fcb 2024-11-20T04:29:34,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/a/dbc20ae3c3654f74a04ab89eb38c8fcb, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T04:29:34,541 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8ea9036ad5b93db8385ce6c5d98c0ad1 in 157ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T04:29:34,543 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/0000000000000000002 2024-11-20T04:29:34,544 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,544 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8ea9036ad5b93db8385ce6c5d98c0ad1 2024-11-20T04:29:34,551 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/8ea9036ad5b93db8385ce6c5d98c0ad1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T04:29:34,553 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8ea9036ad5b93db8385ce6c5d98c0ad1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59740332, jitterRate=-0.10979968309402466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:34,555 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8ea9036ad5b93db8385ce6c5d98c0ad1: Writing region info on filesystem at 1732076974355Initializing all the Stores at 1732076974356 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076974356Obtaining lock to block concurrent updates at 1732076974382 (+26 ms)Preparing flush snapshotting stores in 8ea9036ad5b93db8385ce6c5d98c0ad1 at 1732076974382Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732076974385 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732076974083.8ea9036ad5b93db8385ce6c5d98c0ad1. at 1732076974386 (+1 ms)Flushing 8ea9036ad5b93db8385ce6c5d98c0ad1/a: creating writer at 1732076974387 (+1 ms)Flushing 8ea9036ad5b93db8385ce6c5d98c0ad1/a: appending metadata at 1732076974440 (+53 ms)Flushing 8ea9036ad5b93db8385ce6c5d98c0ad1/a: closing flushed file at 1732076974443 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38467a6: reopening flushed file at 1732076974522 (+79 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8ea9036ad5b93db8385ce6c5d98c0ad1 in 157ms, sequenceid=2, compaction requested=false; wal=null at 1732076974541 (+19 ms)Cleaning up temporary data from old regions at 1732076974544 (+3 ms)Region opened successfully at 1732076974555 (+11 ms) 2024-11-20T04:29:34,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:45776 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56250 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46702 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:45602 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56266 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:42657/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46836 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=701 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=7223 (was 7245) 2024-11-20T04:29:34,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=701, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=7220 2024-11-20T04:29:34,620 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,623 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:34,628 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92824529, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-92824529, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:34,643 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92824529/hregion-92824529.1732076974629, exclude list is [], retry=0 2024-11-20T04:29:34,648 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:34,648 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:34,649 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:34,654 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92824529/hregion-92824529.1732076974629 2024-11-20T04:29:34,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:34,654 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 307a4bf0e374ad59e69902d3f33a19ee, NAME => 'testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:34,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741846_1022 (size=64) 2024-11-20T04:29:34,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741846_1022 (size=64) 2024-11-20T04:29:34,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741846_1022 (size=64) 2024-11-20T04:29:34,671 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:34,673 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,675 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307a4bf0e374ad59e69902d3f33a19ee columnFamilyName a 2024-11-20T04:29:34,676 DEBUG [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:34,676 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] regionserver.HStore(327): Store=307a4bf0e374ad59e69902d3f33a19ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:34,677 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,678 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,679 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,679 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,679 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,689 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:34,690 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 307a4bf0e374ad59e69902d3f33a19ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62861682, jitterRate=-0.06328794360160828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 307a4bf0e374ad59e69902d3f33a19ee: Writing region info on filesystem at 1732076974671Initializing all the Stores at 1732076974672 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076974672Cleaning up temporary data from old regions at 1732076974679 (+7 ms)Region opened successfully at 1732076974690 (+11 ms) 2024-11-20T04:29:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 307a4bf0e374ad59e69902d3f33a19ee, disabling compactions & flushes 2024-11-20T04:29:34,690 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. 2024-11-20T04:29:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. 2024-11-20T04:29:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. after waiting 0 ms 2024-11-20T04:29:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. 2024-11-20T04:29:34,691 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. 2024-11-20T04:29:34,691 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 307a4bf0e374ad59e69902d3f33a19ee: Waiting for close lock at 1732076974690Disabling compacts and flushes for region at 1732076974690Disabling writes for close at 1732076974690Writing region close event to WAL at 1732076974691 (+1 ms)Closed at 1732076974691 2024-11-20T04:29:34,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741845_1021 (size=93) 2024-11-20T04:29:34,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741845_1021 (size=93) 2024-11-20T04:29:34,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741845_1021 (size=93) 2024-11-20T04:29:34,700 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:34,700 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92824529:(num 1732076974629) 2024-11-20T04:29:34,701 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T04:29:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741847_1023 (size=276) 2024-11-20T04:29:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741847_1023 (size=276) 2024-11-20T04:29:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741847_1023 (size=276) 2024-11-20T04:29:34,715 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T04:29:34,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741848_1024 (size=230) 2024-11-20T04:29:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741848_1024 (size=230) 2024-11-20T04:29:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741848_1024 (size=230) 2024-11-20T04:29:34,753 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2, size=230 (230bytes) 2024-11-20T04:29:34,753 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2 2024-11-20T04:29:34,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2 after 1ms 2024-11-20T04:29:34,758 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,758 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2 took 6ms 2024-11-20T04:29:34,761 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2 so closing down 2024-11-20T04:29:34,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:34,763 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T04:29:34,765 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T04:29:34,765 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741849_1025 (size=230) 2024-11-20T04:29:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741849_1025 (size=230) 2024-11-20T04:29:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741849_1025 (size=230) 2024-11-20T04:29:34,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:34,777 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 2024-11-20T04:29:34,777 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-20T04:29:34,777 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2, journal: Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2, size=230 (230bytes) at 1732076974753Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2 so closing down at 1732076974761 (+8 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp at 1732076974765 (+4 ms)3 split writer threads finished at 1732076974765Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732076974775 (+10 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 at 1732076974777 (+2 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732076974777 2024-11-20T04:29:34,792 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1, size=276 (276bytes) 2024-11-20T04:29:34,792 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1 2024-11-20T04:29:34,793 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1 after 1ms 2024-11-20T04:29:34,796 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,796 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1 took 4ms 2024-11-20T04:29:34,799 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1 so closing down 2024-11-20T04:29:34,799 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:34,801 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T04:29:34,803 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T04:29:34,803 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741850_1026 (size=276) 2024-11-20T04:29:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741850_1026 (size=276) 2024-11-20T04:29:34,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741850_1026 (size=276) 2024-11-20T04:29:34,817 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:34,821 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,824 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002, length=230 2024-11-20T04:29:34,826 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 2024-11-20T04:29:34,826 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-20T04:29:34,827 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1, journal: Splitting hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1, size=276 (276bytes) at 1732076974792Finishing writing output for hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1 so closing down at 1732076974799 (+7 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp at 1732076974803 (+4 ms)3 split writer threads finished at 1732076974803Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732076974817 (+14 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 at 1732076974826 (+9 ms)Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732076974826 2024-11-20T04:29:34,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:34,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:34,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal.1732076974830, exclude list is [], retry=0 2024-11-20T04:29:34,853 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:34,853 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:34,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:34,857 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal.1732076974830 2024-11-20T04:29:34,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:34,857 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 307a4bf0e374ad59e69902d3f33a19ee, NAME => 'testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:34,857 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:34,857 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,857 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,860 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,862 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307a4bf0e374ad59e69902d3f33a19ee columnFamilyName a 2024-11-20T04:29:34,862 DEBUG [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:34,863 INFO [StoreOpener-307a4bf0e374ad59e69902d3f33a19ee-1 {}] regionserver.HStore(327): Store=307a4bf0e374ad59e69902d3f33a19ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:34,863 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,864 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,866 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,867 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 2024-11-20T04:29:34,871 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:34,873 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 2024-11-20T04:29:34,873 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 307a4bf0e374ad59e69902d3f33a19ee 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T04:29:34,890 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/.tmp/a/3d20df01e75240e388a26be68aae87a0 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732076974700/Put/seqid=0 2024-11-20T04:29:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741852_1028 (size=5170) 2024-11-20T04:29:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741852_1028 (size=5170) 2024-11-20T04:29:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741852_1028 (size=5170) 2024-11-20T04:29:34,901 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/.tmp/a/3d20df01e75240e388a26be68aae87a0 2024-11-20T04:29:34,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/.tmp/a/3d20df01e75240e388a26be68aae87a0 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/a/3d20df01e75240e388a26be68aae87a0 2024-11-20T04:29:34,921 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/a/3d20df01e75240e388a26be68aae87a0, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T04:29:34,922 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 307a4bf0e374ad59e69902d3f33a19ee in 49ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T04:29:34,923 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/0000000000000000002 2024-11-20T04:29:34,923 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,923 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,926 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 307a4bf0e374ad59e69902d3f33a19ee 2024-11-20T04:29:34,929 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/307a4bf0e374ad59e69902d3f33a19ee/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T04:29:34,931 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 307a4bf0e374ad59e69902d3f33a19ee; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63636337, jitterRate=-0.05174468457698822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:34,931 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 307a4bf0e374ad59e69902d3f33a19ee: Writing region info on filesystem at 1732076974857Initializing all the Stores at 1732076974859 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076974859Obtaining lock to block concurrent updates at 1732076974873 (+14 ms)Preparing flush snapshotting stores in 307a4bf0e374ad59e69902d3f33a19ee at 1732076974873Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732076974873Flushing stores of testReplayEditsWrittenIntoWAL,,1732076974621.307a4bf0e374ad59e69902d3f33a19ee. at 1732076974873Flushing 307a4bf0e374ad59e69902d3f33a19ee/a: creating writer at 1732076974873Flushing 307a4bf0e374ad59e69902d3f33a19ee/a: appending metadata at 1732076974889 (+16 ms)Flushing 307a4bf0e374ad59e69902d3f33a19ee/a: closing flushed file at 1732076974889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@476004fa: reopening flushed file at 1732076974909 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 307a4bf0e374ad59e69902d3f33a19ee in 49ms, sequenceid=2, compaction requested=false; wal=null at 1732076974922 (+13 ms)Cleaning up temporary data from old regions at 1732076974923 (+1 ms)Region opened successfully at 1732076974931 (+8 ms) 2024-11-20T04:29:34,952 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56324 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:45838 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56266 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46920 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46702 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 701) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=7153 (was 7220) 2024-11-20T04:29:34,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=7150 2024-11-20T04:29:34,989 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,991 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:34,992 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:34,996 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87578519, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-87578519, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:35,010 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87578519/hregion-87578519.1732076974996, exclude list is [], retry=0 2024-11-20T04:29:35,014 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:35,015 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:35,015 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:35,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87578519/hregion-87578519.1732076974996 2024-11-20T04:29:35,019 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:35,020 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0c2beebc4aefa25e8bddaa8c27f9db81, NAME => 'testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741854_1030 (size=64) 2024-11-20T04:29:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741854_1030 (size=64) 2024-11-20T04:29:35,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741854_1030 (size=64) 2024-11-20T04:29:35,040 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:35,042 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,044 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName a 2024-11-20T04:29:35,044 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:35,045 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:35,045 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,047 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName b 2024-11-20T04:29:35,047 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:35,048 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:35,048 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,050 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName c 2024-11-20T04:29:35,050 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:35,051 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:35,051 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,055 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,055 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,057 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:35,059 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:35,062 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:35,063 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0c2beebc4aefa25e8bddaa8c27f9db81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74946839, jitterRate=0.11679492890834808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0c2beebc4aefa25e8bddaa8c27f9db81: Writing region info on filesystem at 1732076975040Initializing all the Stores at 1732076975041 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076975041Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076975042 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076975042Cleaning up temporary data from old regions at 1732076975055 (+13 ms)Region opened successfully at 1732076975063 (+8 ms) 2024-11-20T04:29:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0c2beebc4aefa25e8bddaa8c27f9db81, disabling compactions & flushes 2024-11-20T04:29:35,064 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. after waiting 0 ms 2024-11-20T04:29:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:35,065 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:35,065 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0c2beebc4aefa25e8bddaa8c27f9db81: Waiting for close lock at 1732076975064Disabling compacts and flushes for region at 1732076975064Disabling writes for close at 1732076975064Writing region close event to WAL at 1732076975065 (+1 ms)Closed at 1732076975065 2024-11-20T04:29:35,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741853_1029 (size=93) 2024-11-20T04:29:35,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741853_1029 (size=93) 2024-11-20T04:29:35,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741853_1029 (size=93) 2024-11-20T04:29:35,072 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:35,073 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87578519:(num 1732076974996) 2024-11-20T04:29:35,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:35,075 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:35,096 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, exclude list is [], retry=0 2024-11-20T04:29:35,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:35,101 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:35,101 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:35,103 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 2024-11-20T04:29:35,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:35,368 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, size=0 (0bytes) 2024-11-20T04:29:35,368 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 might be still open, length is 0 2024-11-20T04:29:35,368 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 2024-11-20T04:29:35,369 WARN [IPC Server handler 1 on default port 42657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-20T04:29:35,370 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 after 2ms 2024-11-20T04:29:38,189 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46952 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46091:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46952 dst: /127.0.0.1:46091 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46091 remote=/127.0.0.1:46952]. Total timeout mills is 60000, 57131 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:38,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:45860 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45860 dst: /127.0.0.1:43305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:38,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56374 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45985:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56374 dst: /127.0.0.1:45985 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741855_1032 (size=200589) 2024-11-20T04:29:38,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741855_1032 (size=200589) 2024-11-20T04:29:38,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741855_1032 (size=200589) 2024-11-20T04:29:39,142 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T04:29:39,194 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T04:29:39,371 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 after 4003ms 2024-11-20T04:29:39,375 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:39,377 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 took 4009ms 2024-11-20T04:29:39,382 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732076975076.temp 2024-11-20T04:29:39,385 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp 2024-11-20T04:29:39,493 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076; continuing. 2024-11-20T04:29:39,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 so closing down 2024-11-20T04:29:39,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:39,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:39,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741856_1033 (size=200597) 2024-11-20T04:29:39,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741856_1033 (size=200597) 2024-11-20T04:29:39,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741856_1033 (size=200597) 2024-11-20T04:29:39,500 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-20T04:29:39,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 2024-11-20T04:29:39,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T04:29:39,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, journal: Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, size=0 (0bytes) at 1732076975368Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp at 1732076979385 (+4017 ms)Split 1024 edits, skipped 0 edits. at 1732076979437 (+52 ms)Split 2048 edits, skipped 0 edits. at 1732076979466 (+29 ms)Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 so closing down at 1732076979494 (+28 ms)3 split writer threads finished at 1732076979494Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1732076979500 (+6 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000000001-wal.1732076975076.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 at 1732076979502 (+2 ms)Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076, size=0, length=0, corrupted=false, cancelled=false at 1732076979502 2024-11-20T04:29:39,505 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076975076 2024-11-20T04:29:39,506 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 2024-11-20T04:29:39,506 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:39,509 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:39,523 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076979509, exclude list is [], retry=0 2024-11-20T04:29:39,527 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:39,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:39,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:39,532 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076979509 2024-11-20T04:29:39,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:39,533 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:39,538 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,540 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName a 2024-11-20T04:29:39,541 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:39,542 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:39,542 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,544 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName b 2024-11-20T04:29:39,544 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:39,544 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:39,545 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,546 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c2beebc4aefa25e8bddaa8c27f9db81 columnFamilyName c 2024-11-20T04:29:39,546 DEBUG [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:39,546 INFO [StoreOpener-0c2beebc4aefa25e8bddaa8c27f9db81-1 {}] regionserver.HStore(327): Store=0c2beebc4aefa25e8bddaa8c27f9db81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:39,547 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,548 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,550 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:39,551 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 2024-11-20T04:29:39,555 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:39,607 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T04:29:40,022 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c2beebc4aefa25e8bddaa8c27f9db81 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T04:29:40,061 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/0c939e4c07e041bcbd3d62e136b44f54 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732076975113/Put/seqid=0 2024-11-20T04:29:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741858_1035 (size=50463) 2024-11-20T04:29:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741858_1035 (size=50463) 2024-11-20T04:29:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741858_1035 (size=50463) 2024-11-20T04:29:40,080 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/0c939e4c07e041bcbd3d62e136b44f54 2024-11-20T04:29:40,088 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/0c939e4c07e041bcbd3d62e136b44f54 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/a/0c939e4c07e041bcbd3d62e136b44f54 2024-11-20T04:29:40,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/a/0c939e4c07e041bcbd3d62e136b44f54, entries=754, sequenceid=754, filesize=49.3 K 2024-11-20T04:29:40,095 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 0c2beebc4aefa25e8bddaa8c27f9db81 in 73ms, sequenceid=754, compaction requested=false; wal=null 2024-11-20T04:29:40,121 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T04:29:40,121 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c2beebc4aefa25e8bddaa8c27f9db81 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T04:29:40,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/810b33f466cc407585bdf8a33c1909c3 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732076975169/Put/seqid=0 2024-11-20T04:29:40,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741859_1036 (size=20072) 2024-11-20T04:29:40,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741859_1036 (size=20072) 2024-11-20T04:29:40,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741859_1036 (size=20072) 2024-11-20T04:29:40,147 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/810b33f466cc407585bdf8a33c1909c3 2024-11-20T04:29:40,184 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/55546b79014e4697bf5aff8b22c74b59 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732076975196/Put/seqid=0 2024-11-20T04:29:40,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741860_1037 (size=35835) 2024-11-20T04:29:40,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741860_1037 (size=35835) 2024-11-20T04:29:40,194 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/55546b79014e4697bf5aff8b22c74b59 2024-11-20T04:29:40,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741860_1037 (size=35835) 2024-11-20T04:29:40,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/a/810b33f466cc407585bdf8a33c1909c3 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/a/810b33f466cc407585bdf8a33c1909c3 2024-11-20T04:29:40,211 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/a/810b33f466cc407585bdf8a33c1909c3, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-20T04:29:40,212 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/55546b79014e4697bf5aff8b22c74b59 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/b/55546b79014e4697bf5aff8b22c74b59 2024-11-20T04:29:40,221 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/b/55546b79014e4697bf5aff8b22c74b59, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-20T04:29:40,221 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0c2beebc4aefa25e8bddaa8c27f9db81 in 100ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-20T04:29:40,236 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T04:29:40,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c2beebc4aefa25e8bddaa8c27f9db81 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T04:29:40,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/3332b5c7bff24687b83fe35a348d369d is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732076975221/Put/seqid=0 2024-11-20T04:29:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741861_1038 (size=35082) 2024-11-20T04:29:40,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741861_1038 (size=35082) 2024-11-20T04:29:40,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741861_1038 (size=35082) 2024-11-20T04:29:40,262 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/3332b5c7bff24687b83fe35a348d369d 2024-11-20T04:29:40,298 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/a17d46330cc442fe8bb464296524f0ee is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732076975261/Put/seqid=0 2024-11-20T04:29:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741862_1039 (size=20825) 2024-11-20T04:29:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741862_1039 (size=20825) 2024-11-20T04:29:40,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741862_1039 (size=20825) 2024-11-20T04:29:40,309 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/a17d46330cc442fe8bb464296524f0ee 2024-11-20T04:29:40,318 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/b/3332b5c7bff24687b83fe35a348d369d as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/b/3332b5c7bff24687b83fe35a348d369d 2024-11-20T04:29:40,327 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/b/3332b5c7bff24687b83fe35a348d369d, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-20T04:29:40,329 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/a17d46330cc442fe8bb464296524f0ee as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/c/a17d46330cc442fe8bb464296524f0ee 2024-11-20T04:29:40,336 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/c/a17d46330cc442fe8bb464296524f0ee, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-20T04:29:40,337 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0c2beebc4aefa25e8bddaa8c27f9db81 in 100ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-20T04:29:40,351 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732076975318/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:40,356 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 2024-11-20T04:29:40,356 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T04:29:40,357 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c2beebc4aefa25e8bddaa8c27f9db81 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-20T04:29:40,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/5358d3bc1db24167bce4ab7c38f79deb is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732076975272/Put/seqid=0 2024-11-20T04:29:40,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741863_1040 (size=50301) 2024-11-20T04:29:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741863_1040 (size=50301) 2024-11-20T04:29:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741863_1040 (size=50301) 2024-11-20T04:29:40,380 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/5358d3bc1db24167bce4ab7c38f79deb 2024-11-20T04:29:40,388 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5358d3bc1db24167bce4ab7c38f79deb 2024-11-20T04:29:40,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/.tmp/c/5358d3bc1db24167bce4ab7c38f79deb as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/c/5358d3bc1db24167bce4ab7c38f79deb 2024-11-20T04:29:40,396 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5358d3bc1db24167bce4ab7c38f79deb 2024-11-20T04:29:40,397 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/c/5358d3bc1db24167bce4ab7c38f79deb, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-20T04:29:40,397 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 0c2beebc4aefa25e8bddaa8c27f9db81 in 41ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-20T04:29:40,398 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/0000000000000003002 2024-11-20T04:29:40,399 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:40,399 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:40,399 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T04:29:40,401 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0c2beebc4aefa25e8bddaa8c27f9db81 2024-11-20T04:29:40,405 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenIntoWAL/0c2beebc4aefa25e8bddaa8c27f9db81/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-20T04:29:40,406 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0c2beebc4aefa25e8bddaa8c27f9db81; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75199134, jitterRate=0.12055441737174988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T04:29:40,406 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0c2beebc4aefa25e8bddaa8c27f9db81: Writing region info on filesystem at 1732076979533Initializing all the Stores at 1732076979535 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076979535Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076979538 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076979538Cleaning up temporary data from old regions at 1732076980399 (+861 ms)Region opened successfully at 1732076980406 (+7 ms) 2024-11-20T04:29:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0c2beebc4aefa25e8bddaa8c27f9db81, disabling compactions & flushes 2024-11-20T04:29:40,473 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. after waiting 0 ms 2024-11-20T04:29:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:40,475 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732076974990.0c2beebc4aefa25e8bddaa8c27f9db81. 2024-11-20T04:29:40,476 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0c2beebc4aefa25e8bddaa8c27f9db81: Waiting for close lock at 1732076980473Disabling compacts and flushes for region at 1732076980473Disabling writes for close at 1732076980473Writing region close event to WAL at 1732076980475 (+2 ms)Closed at 1732076980475 2024-11-20T04:29:40,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741857_1034 (size=93) 2024-11-20T04:29:40,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741857_1034 (size=93) 2024-11-20T04:29:40,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741857_1034 (size=93) 2024-11-20T04:29:40,485 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:40,485 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732076979509) 2024-11-20T04:29:40,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361531271_22 at /127.0.0.1:47026 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36549 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:40331 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:36549 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:42657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361531271_22 at /127.0.0.1:45902 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:42657 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361531271_22 at /127.0.0.1:56428 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40331 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361531271_22 at /127.0.0.1:56414 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=249 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=7074 (was 7150) 2024-11-20T04:29:40,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=249, ProcessCount=11, AvailableMemoryMB=7073 2024-11-20T04:29:40,534 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:40,537 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:40,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:40,541 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31013409, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-31013409, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:40,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31013409/hregion-31013409.1732076980542, exclude list is [], retry=0 2024-11-20T04:29:40,561 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:40,561 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:40,561 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:40,567 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31013409/hregion-31013409.1732076980542 2024-11-20T04:29:40,567 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:40,567 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 40f87862317f2da69e56fada57ab9945, NAME => 'test2727,,1732076980535.40f87862317f2da69e56fada57ab9945.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:40,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741865_1042 (size=43) 2024-11-20T04:29:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741865_1042 (size=43) 2024-11-20T04:29:40,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741865_1042 (size=43) 2024-11-20T04:29:40,582 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732076980535.40f87862317f2da69e56fada57ab9945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:40,584 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,586 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName a 2024-11-20T04:29:40,586 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:40,587 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:40,587 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,589 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName b 2024-11-20T04:29:40,589 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:40,589 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:40,590 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,591 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName c 2024-11-20T04:29:40,592 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:40,592 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:40,592 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,593 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,594 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,595 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,595 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,596 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:40,598 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:40,603 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:40,603 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40f87862317f2da69e56fada57ab9945; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60470697, jitterRate=-0.09891639649868011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40f87862317f2da69e56fada57ab9945: Writing region info on filesystem at 1732076980582Initializing all the Stores at 1732076980583 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076980583Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076980583Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076980583Cleaning up temporary data from old regions at 1732076980595 (+12 ms)Region opened successfully at 1732076980604 (+9 ms) 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40f87862317f2da69e56fada57ab9945, disabling compactions & flushes 2024-11-20T04:29:40,605 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. after waiting 0 ms 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:40,605 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:40,605 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40f87862317f2da69e56fada57ab9945: Waiting for close lock at 1732076980605Disabling compacts and flushes for region at 1732076980605Disabling writes for close at 1732076980605Writing region close event to WAL at 1732076980605Closed at 1732076980605 2024-11-20T04:29:40,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741864_1041 (size=93) 2024-11-20T04:29:40,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741864_1041 (size=93) 2024-11-20T04:29:40,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741864_1041 (size=93) 2024-11-20T04:29:40,614 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:40,614 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31013409:(num 1732076980542) 2024-11-20T04:29:40,614 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:40,617 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:40,631 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, exclude list is [], retry=0 2024-11-20T04:29:40,635 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:40,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:40,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:40,643 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 2024-11-20T04:29:40,644 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741866_1043 (size=200357) 2024-11-20T04:29:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741866_1043 (size=200357) 2024-11-20T04:29:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741866_1043 (size=200357) 2024-11-20T04:29:40,832 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, size=195.7 K (200357bytes) 2024-11-20T04:29:40,832 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 2024-11-20T04:29:40,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 after 1ms 2024-11-20T04:29:40,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:40,838 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 took 6ms 2024-11-20T04:29:40,845 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732076980617.temp 2024-11-20T04:29:40,847 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp 2024-11-20T04:29:40,908 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 so closing down 2024-11-20T04:29:40,908 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:40,909 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741867_1044 (size=200357) 2024-11-20T04:29:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741867_1044 (size=200357) 2024-11-20T04:29:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741867_1044 (size=200357) 2024-11-20T04:29:40,917 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp (wrote 3000 edits, skipped 0 edits in 24 ms) 2024-11-20T04:29:40,919 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp to hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 2024-11-20T04:29:40,920 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-20T04:29:40,920 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, journal: Splitting hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, size=195.7 K (200357bytes) at 1732076980832Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp at 1732076980847 (+15 ms)Split 1024 edits, skipped 0 edits. at 1732076980866 (+19 ms)Split 2048 edits, skipped 0 edits. at 1732076980889 (+23 ms)Finishing writing output for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 so closing down at 1732076980908 (+19 ms)3 split writer threads finished at 1732076980909 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp (wrote 3000 edits, skipped 0 edits in 24 ms) at 1732076980917 (+8 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000000001-wal.1732076980617.temp to hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 at 1732076980919 (+2 ms)Processed 3000 edits across 1 Regions in 81 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732076980920 (+1 ms) 2024-11-20T04:29:40,922 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980617 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076980617 2024-11-20T04:29:40,923 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 2024-11-20T04:29:40,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:40,926 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:40,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, exclude list is [], retry=0 2024-11-20T04:29:40,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:40,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:40,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:40,950 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 2024-11-20T04:29:40,952 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:40,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T04:29:40,996 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:40,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T04:29:40,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T04:29:40,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T04:29:40,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:41,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-20T04:29:41,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:41,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741868_1045 (size=200484) 2024-11-20T04:29:41,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741868_1045 (size=200484) 2024-11-20T04:29:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741868_1045 (size=200484) 2024-11-20T04:29:41,119 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, size=195.8 K (200484bytes) 2024-11-20T04:29:41,119 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 2024-11-20T04:29:41,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 after 1ms 2024-11-20T04:29:41,123 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:41,125 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 took 6ms 2024-11-20T04:29:41,130 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732076980927.temp 2024-11-20T04:29:41,133 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp 2024-11-20T04:29:41,190 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 so closing down 2024-11-20T04:29:41,190 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:41,191 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741869_1046 (size=200484) 2024-11-20T04:29:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741869_1046 (size=200484) 2024-11-20T04:29:41,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741869_1046 (size=200484) 2024-11-20T04:29:41,196 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp (wrote 3000 edits, skipped 0 edits in 22 ms) 2024-11-20T04:29:41,198 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp to hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 2024-11-20T04:29:41,198 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-20T04:29:41,198 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, journal: Splitting hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, size=195.8 K (200484bytes) at 1732076981119Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp at 1732076981133 (+14 ms)Split 1024 edits, skipped 0 edits. at 1732076981150 (+17 ms)Split 2048 edits, skipped 0 edits. at 1732076981171 (+21 ms)Finishing writing output for hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 so closing down at 1732076981190 (+19 ms)3 split writer threads finished at 1732076981191 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp (wrote 3000 edits, skipped 0 edits in 22 ms) at 1732076981196 (+5 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003001-wal.1732076980927.temp to hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 at 1732076981198 (+2 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732076981198 2024-11-20T04:29:41,202 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076980927 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076980927 2024-11-20T04:29:41,203 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 2024-11-20T04:29:41,203 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:41,205 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/test2727-manual,16010,1732076980533, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:41,222 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076981206, exclude list is [], retry=0 2024-11-20T04:29:41,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:41,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:41,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:41,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732076980533/wal.1732076981206 2024-11-20T04:29:41,230 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:41,230 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40f87862317f2da69e56fada57ab9945, NAME => 'test2727,,1732076980535.40f87862317f2da69e56fada57ab9945.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:41,231 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732076980535.40f87862317f2da69e56fada57ab9945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:41,231 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,231 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,233 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,234 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName a 2024-11-20T04:29:41,234 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,235 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,235 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,236 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName b 2024-11-20T04:29:41,236 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,237 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,237 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,238 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f87862317f2da69e56fada57ab9945 columnFamilyName c 2024-11-20T04:29:41,239 DEBUG [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,239 INFO [StoreOpener-40f87862317f2da69e56fada57ab9945-1 {}] regionserver.HStore(327): Store=40f87862317f2da69e56fada57ab9945/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,240 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,240 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,244 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 2024-11-20T04:29:41,247 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:41,297 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 2024-11-20T04:29:41,298 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 2024-11-20T04:29:41,301 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:41,350 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 2024-11-20T04:29:41,351 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40f87862317f2da69e56fada57ab9945 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-20T04:29:41,374 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/a/5e1ad53c832b4847a4aca7e100ac3eb1 is 41, key is test2727/a:100/1732076980957/Put/seqid=0 2024-11-20T04:29:41,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741871_1048 (size=84227) 2024-11-20T04:29:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741871_1048 (size=84227) 2024-11-20T04:29:41,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741871_1048 (size=84227) 2024-11-20T04:29:41,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/a/5e1ad53c832b4847a4aca7e100ac3eb1 2024-11-20T04:29:41,422 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/b/a083dcc499ea4127b8dd8c051f45b986 is 41, key is test2727/b:100/1732076981008/Put/seqid=0 2024-11-20T04:29:41,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741872_1049 (size=84609) 2024-11-20T04:29:41,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741872_1049 (size=84609) 2024-11-20T04:29:41,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741872_1049 (size=84609) 2024-11-20T04:29:41,429 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/b/a083dcc499ea4127b8dd8c051f45b986 2024-11-20T04:29:41,457 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/c/a53fdd63ca3b43888332a6c7d8441a65 is 41, key is test2727/c:100/1732076981054/Put/seqid=0 2024-11-20T04:29:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741873_1050 (size=84609) 2024-11-20T04:29:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741873_1050 (size=84609) 2024-11-20T04:29:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741873_1050 (size=84609) 2024-11-20T04:29:41,465 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/c/a53fdd63ca3b43888332a6c7d8441a65 2024-11-20T04:29:41,473 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/a/5e1ad53c832b4847a4aca7e100ac3eb1 as hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/a/5e1ad53c832b4847a4aca7e100ac3eb1 2024-11-20T04:29:41,481 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/a/5e1ad53c832b4847a4aca7e100ac3eb1, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-20T04:29:41,482 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/b/a083dcc499ea4127b8dd8c051f45b986 as hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/b/a083dcc499ea4127b8dd8c051f45b986 2024-11-20T04:29:41,492 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/b/a083dcc499ea4127b8dd8c051f45b986, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T04:29:41,494 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/.tmp/c/a53fdd63ca3b43888332a6c7d8441a65 as hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/c/a53fdd63ca3b43888332a6c7d8441a65 2024-11-20T04:29:41,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/c/a53fdd63ca3b43888332a6c7d8441a65, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T04:29:41,501 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 40f87862317f2da69e56fada57ab9945 in 150ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-20T04:29:41,502 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000003000 2024-11-20T04:29:41,502 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/0000000000000006000 2024-11-20T04:29:41,504 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,504 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,504 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:41,506 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40f87862317f2da69e56fada57ab9945 2024-11-20T04:29:41,509 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/test2727/40f87862317f2da69e56fada57ab9945/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-20T04:29:41,510 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40f87862317f2da69e56fada57ab9945; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69747518, jitterRate=0.039319008588790894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:41,511 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40f87862317f2da69e56fada57ab9945: Writing region info on filesystem at 1732076981231Initializing all the Stores at 1732076981232 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981233 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981233Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981233Obtaining lock to block concurrent updates at 1732076981351 (+118 ms)Preparing flush snapshotting stores in 40f87862317f2da69e56fada57ab9945 at 1732076981351Finished memstore snapshotting test2727,,1732076980535.40f87862317f2da69e56fada57ab9945., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732076981351Flushing stores of test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. at 1732076981351Flushing 40f87862317f2da69e56fada57ab9945/a: creating writer at 1732076981351Flushing 40f87862317f2da69e56fada57ab9945/a: appending metadata at 1732076981373 (+22 ms)Flushing 40f87862317f2da69e56fada57ab9945/a: closing flushed file at 1732076981373Flushing 40f87862317f2da69e56fada57ab9945/b: creating writer at 1732076981397 (+24 ms)Flushing 40f87862317f2da69e56fada57ab9945/b: appending metadata at 1732076981420 (+23 ms)Flushing 40f87862317f2da69e56fada57ab9945/b: closing flushed file at 1732076981420Flushing 40f87862317f2da69e56fada57ab9945/c: creating writer at 1732076981436 (+16 ms)Flushing 40f87862317f2da69e56fada57ab9945/c: appending metadata at 1732076981456 (+20 ms)Flushing 40f87862317f2da69e56fada57ab9945/c: closing flushed file at 1732076981456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b367bcf: reopening flushed file at 1732076981472 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72267229: reopening flushed file at 1732076981481 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bcb79a3: reopening flushed file at 1732076981492 (+11 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 40f87862317f2da69e56fada57ab9945 in 150ms, sequenceid=6000, compaction requested=false; wal=null at 1732076981501 (+9 ms)Cleaning up temporary data from old regions at 1732076981504 (+3 ms)Region opened successfully at 1732076981511 (+7 ms) 2024-11-20T04:29:41,513 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-20T04:29:41,513 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40f87862317f2da69e56fada57ab9945, disabling compactions & flushes 2024-11-20T04:29:41,513 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:41,513 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:41,513 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. after waiting 0 ms 2024-11-20T04:29:41,513 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:41,515 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732076980535.40f87862317f2da69e56fada57ab9945. 2024-11-20T04:29:41,515 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40f87862317f2da69e56fada57ab9945: Waiting for close lock at 1732076981513Disabling compacts and flushes for region at 1732076981513Disabling writes for close at 1732076981513Writing region close event to WAL at 1732076981515 (+2 ms)Closed at 1732076981515 2024-11-20T04:29:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741870_1047 (size=93) 2024-11-20T04:29:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741870_1047 (size=93) 2024-11-20T04:29:41,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741870_1047 (size=93) 2024-11-20T04:29:41,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:41,523 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732076981206) 2024-11-20T04:29:41,539 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=404 (was 400) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:50284 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51732 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:39212 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 249) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6872 (was 7073) 2024-11-20T04:29:41,551 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=6871 2024-11-20T04:29:41,567 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:41,575 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:41,576 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732076981575 2024-11-20T04:29:41,583 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 2024-11-20T04:29:41,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:41,586 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 32e73dd8f7e69783b77290cf916ba6c9, NAME => 'testSequentialEditLogSeqNum,,1732076981568.32e73dd8f7e69783b77290cf916ba6c9.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:41,586 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732076981568.32e73dd8f7e69783b77290cf916ba6c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:41,586 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,586 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,587 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9 doesn't exist for region: 32e73dd8f7e69783b77290cf916ba6c9 on table testSequentialEditLogSeqNum 2024-11-20T04:29:41,588 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 32e73dd8f7e69783b77290cf916ba6c9 on table testSequentialEditLogSeqNum 2024-11-20T04:29:41,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741875_1052 (size=62) 2024-11-20T04:29:41,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741875_1052 (size=62) 2024-11-20T04:29:41,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741875_1052 (size=62) 2024-11-20T04:29:41,603 INFO [StoreOpener-32e73dd8f7e69783b77290cf916ba6c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,605 INFO [StoreOpener-32e73dd8f7e69783b77290cf916ba6c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32e73dd8f7e69783b77290cf916ba6c9 columnFamilyName a 2024-11-20T04:29:41,605 DEBUG [StoreOpener-32e73dd8f7e69783b77290cf916ba6c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,606 INFO [StoreOpener-32e73dd8f7e69783b77290cf916ba6c9-1 {}] regionserver.HStore(327): Store=32e73dd8f7e69783b77290cf916ba6c9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,606 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,608 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,608 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,610 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 32e73dd8f7e69783b77290cf916ba6c9 2024-11-20T04:29:41,613 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:41,614 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 32e73dd8f7e69783b77290cf916ba6c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69157509, jitterRate=0.030527189373970032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:41,616 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 32e73dd8f7e69783b77290cf916ba6c9: Writing region info on filesystem at 1732076981587Initializing all the Stores at 1732076981602 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981603 (+1 ms)Cleaning up temporary data from old regions at 1732076981608 (+5 ms)Region opened successfully at 1732076981616 (+8 ms) 2024-11-20T04:29:41,634 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 32e73dd8f7e69783b77290cf916ba6c9 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-20T04:29:41,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/.tmp/a/a257f9da94924e1f8104c47caa612410 is 81, key is testSequentialEditLogSeqNum/a:x0/1732076981616/Put/seqid=0 2024-11-20T04:29:41,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741876_1053 (size=5833) 2024-11-20T04:29:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741876_1053 (size=5833) 2024-11-20T04:29:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741876_1053 (size=5833) 2024-11-20T04:29:41,680 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/.tmp/a/a257f9da94924e1f8104c47caa612410 2024-11-20T04:29:41,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/.tmp/a/a257f9da94924e1f8104c47caa612410 as hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/a/a257f9da94924e1f8104c47caa612410 2024-11-20T04:29:41,696 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/a/a257f9da94924e1f8104c47caa612410, entries=10, sequenceid=13, filesize=5.7 K 2024-11-20T04:29:41,698 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 32e73dd8f7e69783b77290cf916ba6c9 in 64ms, sequenceid=13, compaction requested=false 2024-11-20T04:29:41,698 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 32e73dd8f7e69783b77290cf916ba6c9: 2024-11-20T04:29:41,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T04:29:41,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T04:29:41,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T04:29:41,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T04:29:41,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T04:29:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741874_1051 (size=1616) 2024-11-20T04:29:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741874_1051 (size=1616) 2024-11-20T04:29:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741874_1051 (size=1616) 2024-11-20T04:29:41,738 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575, size=1.6 K (1616bytes) 2024-11-20T04:29:41,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 2024-11-20T04:29:41,739 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 after 1ms 2024-11-20T04:29:41,742 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:41,743 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 took 5ms 2024-11-20T04:29:41,746 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 so closing down 2024-11-20T04:29:41,746 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:41,747 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732076981575.temp 2024-11-20T04:29:41,749 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp 2024-11-20T04:29:41,750 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:41,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741877_1054 (size=1175) 2024-11-20T04:29:41,764 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:41,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741877_1054 (size=1175) 2024-11-20T04:29:41,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741877_1054 (size=1175) 2024-11-20T04:29:41,766 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp to hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000020 2024-11-20T04:29:41,766 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-20T04:29:41,766 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575, journal: Splitting hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575, size=1.6 K (1616bytes) at 1732076981738Finishing writing output for hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575 so closing down at 1732076981746 (+8 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp at 1732076981750 (+4 ms)3 split writer threads finished at 1732076981750Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732076981764 (+14 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000003-wal.1732076981575.temp to hdfs://localhost:42657/hbase/data/default/testSequentialEditLogSeqNum/32e73dd8f7e69783b77290cf916ba6c9/recovered.edits/0000000000000000020 at 1732076981766 (+2 ms)Processed 17 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732076981567/wal.1732076981575, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732076981766 2024-11-20T04:29:41,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:50284 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51732 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6871) - AvailableMemoryMB LEAK? - 2024-11-20T04:29:41,803 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=6875 2024-11-20T04:29:41,827 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:41,830 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:41,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:41,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80654745, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-80654745, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:41,872 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80654745/hregion-80654745.1732076981853, exclude list is [], retry=0 2024-11-20T04:29:41,876 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:41,876 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:41,877 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:41,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80654745/hregion-80654745.1732076981853 2024-11-20T04:29:41,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:41,932 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b316137876d611e4ec3a21222f259547, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741879_1056 (size=70) 2024-11-20T04:29:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741879_1056 (size=70) 2024-11-20T04:29:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741879_1056 (size=70) 2024-11-20T04:29:41,963 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:41,965 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,981 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName a 2024-11-20T04:29:41,982 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,982 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,983 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,984 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName b 2024-11-20T04:29:41,984 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,985 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,985 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,986 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName c 2024-11-20T04:29:41,986 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:41,986 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:41,987 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,988 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,988 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,989 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,989 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,990 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:41,991 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:41,994 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:41,994 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b316137876d611e4ec3a21222f259547; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63397851, jitterRate=-0.05529840290546417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:41,995 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b316137876d611e4ec3a21222f259547: Writing region info on filesystem at 1732076981963Initializing all the Stores at 1732076981965 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981965Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981965Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076981965Cleaning up temporary data from old regions at 1732076981989 (+24 ms)Region opened successfully at 1732076981995 (+6 ms) 2024-11-20T04:29:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b316137876d611e4ec3a21222f259547, disabling compactions & flushes 2024-11-20T04:29:41,996 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. after waiting 0 ms 2024-11-20T04:29:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:41,996 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b316137876d611e4ec3a21222f259547: Waiting for close lock at 1732076981995Disabling compacts and flushes for region at 1732076981995Disabling writes for close at 1732076981996 (+1 ms)Writing region close event to WAL at 1732076981996Closed at 1732076981996 2024-11-20T04:29:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741878_1055 (size=93) 2024-11-20T04:29:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741878_1055 (size=93) 2024-11-20T04:29:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741878_1055 (size=93) 2024-11-20T04:29:42,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:42,011 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80654745:(num 1732076981853) 2024-11-20T04:29:42,011 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:42,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:42,029 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, exclude list is [], retry=0 2024-11-20T04:29:42,032 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:42,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:42,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:42,035 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 2024-11-20T04:29:42,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b316137876d611e4ec3a21222f259547, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,038 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,039 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName a 2024-11-20T04:29:42,039 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:42,040 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:42,040 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,041 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName b 2024-11-20T04:29:42,041 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:42,041 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:42,041 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,042 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName c 2024-11-20T04:29:42,042 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:42,043 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:42,043 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,046 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,046 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,046 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:42,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:42,049 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b316137876d611e4ec3a21222f259547; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75265236, jitterRate=0.1215394139289856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:42,050 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b316137876d611e4ec3a21222f259547: Writing region info on filesystem at 1732076982036Initializing all the Stores at 1732076982037 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076982037Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076982037Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076982037Cleaning up temporary data from old regions at 1732076982046 (+9 ms)Region opened successfully at 1732076982050 (+4 ms) 2024-11-20T04:29:42,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732076982054/Put/seqid=0 2024-11-20T04:29:42,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741881_1058 (size=4826) 2024-11-20T04:29:42,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741881_1058 (size=4826) 2024-11-20T04:29:42,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741881_1058 (size=4826) 2024-11-20T04:29:42,065 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b316137876d611e4ec3a21222f259547/a 2024-11-20T04:29:42,073 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-20T04:29:42,073 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T04:29:42,073 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b316137876d611e4ec3a21222f259547: 2024-11-20T04:29:42,075 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/20e086efb7894e0c97a8f350e40db17a_SeqId_3_ 2024-11-20T04:29:42,076 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b316137876d611e4ec3a21222f259547/a as hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/20e086efb7894e0c97a8f350e40db17a_SeqId_3_ - updating store file list. 2024-11-20T04:29:42,081 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 20e086efb7894e0c97a8f350e40db17a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T04:29:42,081 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/20e086efb7894e0c97a8f350e40db17a_SeqId_3_ into b316137876d611e4ec3a21222f259547/a 2024-11-20T04:29:42,081 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42657/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b316137876d611e4ec3a21222f259547/a (new location: hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/20e086efb7894e0c97a8f350e40db17a_SeqId_3_) 2024-11-20T04:29:42,124 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, size=0 (0bytes) 2024-11-20T04:29:42,124 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 might be still open, length is 0 2024-11-20T04:29:42,124 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 2024-11-20T04:29:42,125 WARN [IPC Server handler 4 on default port 42657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-20T04:29:42,128 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 after 4ms 2024-11-20T04:29:44,387 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:39314 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46091:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39314 dst: /127.0.0.1:46091 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46091 remote=/127.0.0.1:39314]. Total timeout mills is 60000, 57700 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:44,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:50356 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45985:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50356 dst: /127.0.0.1:45985 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:44,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51812 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51812 dst: /127.0.0.1:43305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741880_1059 (size=434) 2024-11-20T04:29:44,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741880_1059 (size=434) 2024-11-20T04:29:46,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 after 4005ms 2024-11-20T04:29:46,133 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:46,134 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 took 4009ms 2024-11-20T04:29:46,135 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014; continuing. 2024-11-20T04:29:46,136 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 so closing down 2024-11-20T04:29:46,136 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:46,138 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732076982014.temp 2024-11-20T04:29:46,139 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp 2024-11-20T04:29:46,139 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741882_1060 (size=236) 2024-11-20T04:29:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741882_1060 (size=236) 2024-11-20T04:29:46,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741882_1060 (size=236) 2024-11-20T04:29:46,149 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:46,150 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp to hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 2024-11-20T04:29:46,151 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T04:29:46,151 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, journal: Splitting hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, size=0 (0bytes) at 1732076982124Finishing writing output for hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 so closing down at 1732076986136 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp at 1732076986139 (+3 ms)3 split writer threads finished at 1732076986139Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732076986149 (+10 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005-wal.1732076982014.temp to hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 at 1732076986150 (+1 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014, size=0, length=0, corrupted=false, cancelled=false at 1732076986151 (+1 ms) 2024-11-20T04:29:46,153 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076982014 2024-11-20T04:29:46,153 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 2024-11-20T04:29:46,154 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:46,155 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:46,169 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076986156, exclude list is [], retry=0 2024-11-20T04:29:46,172 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:46,172 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:46,172 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:46,174 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076986156 2024-11-20T04:29:46,175 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:46,175 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b316137876d611e4ec3a21222f259547, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:46,175 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:46,175 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,175 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,177 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,178 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName a 2024-11-20T04:29:46,178 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,185 DEBUG [StoreFileOpener-b316137876d611e4ec3a21222f259547-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 20e086efb7894e0c97a8f350e40db17a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T04:29:46,185 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/20e086efb7894e0c97a8f350e40db17a_SeqId_3_ 2024-11-20T04:29:46,186 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:46,186 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,187 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName b 2024-11-20T04:29:46,187 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,187 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:46,187 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,188 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b316137876d611e4ec3a21222f259547 columnFamilyName c 2024-11-20T04:29:46,188 DEBUG [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,189 INFO [StoreOpener-b316137876d611e4ec3a21222f259547-1 {}] regionserver.HStore(327): Store=b316137876d611e4ec3a21222f259547/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:46,189 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,190 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,192 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 2024-11-20T04:29:46,194 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:46,195 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 2024-11-20T04:29:46,196 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b316137876d611e4ec3a21222f259547 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-20T04:29:46,215 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/.tmp/a/1224fc15771a4aff9ff6e0fe5fdf0c41 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732076982086/Put/seqid=0 2024-11-20T04:29:46,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741884_1062 (size=5149) 2024-11-20T04:29:46,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741884_1062 (size=5149) 2024-11-20T04:29:46,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741884_1062 (size=5149) 2024-11-20T04:29:46,223 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/.tmp/a/1224fc15771a4aff9ff6e0fe5fdf0c41 2024-11-20T04:29:46,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/.tmp/a/1224fc15771a4aff9ff6e0fe5fdf0c41 as hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/1224fc15771a4aff9ff6e0fe5fdf0c41 2024-11-20T04:29:46,235 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/a/1224fc15771a4aff9ff6e0fe5fdf0c41, entries=1, sequenceid=5, filesize=5.0 K 2024-11-20T04:29:46,236 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b316137876d611e4ec3a21222f259547 in 41ms, sequenceid=5, compaction requested=false; wal=null 2024-11-20T04:29:46,237 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/0000000000000000005 2024-11-20T04:29:46,238 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,238 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,239 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:46,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b316137876d611e4ec3a21222f259547 2024-11-20T04:29:46,244 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b316137876d611e4ec3a21222f259547/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-20T04:29:46,245 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b316137876d611e4ec3a21222f259547; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70032125, jitterRate=0.04355998337268829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:46,245 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b316137876d611e4ec3a21222f259547: Writing region info on filesystem at 1732076986176Initializing all the Stores at 1732076986177 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076986177Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076986177Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076986177Obtaining lock to block concurrent updates at 1732076986196 (+19 ms)Preparing flush snapshotting stores in b316137876d611e4ec3a21222f259547 at 1732076986196Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732076986196Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. at 1732076986196Flushing b316137876d611e4ec3a21222f259547/a: creating writer at 1732076986196Flushing b316137876d611e4ec3a21222f259547/a: appending metadata at 1732076986214 (+18 ms)Flushing b316137876d611e4ec3a21222f259547/a: closing flushed file at 1732076986214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5899ba26: reopening flushed file at 1732076986228 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b316137876d611e4ec3a21222f259547 in 41ms, sequenceid=5, compaction requested=false; wal=null at 1732076986236 (+8 ms)Cleaning up temporary data from old regions at 1732076986238 (+2 ms)Region opened successfully at 1732076986245 (+7 ms) 2024-11-20T04:29:46,250 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b316137876d611e4ec3a21222f259547, disabling compactions & flushes 2024-11-20T04:29:46,250 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:46,250 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:46,250 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. after waiting 0 ms 2024-11-20T04:29:46,250 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:46,251 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732076981828.b316137876d611e4ec3a21222f259547. 2024-11-20T04:29:46,251 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b316137876d611e4ec3a21222f259547: Waiting for close lock at 1732076986250Disabling compacts and flushes for region at 1732076986250Disabling writes for close at 1732076986250Writing region close event to WAL at 1732076986251 (+1 ms)Closed at 1732076986251 2024-11-20T04:29:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741883_1061 (size=93) 2024-11-20T04:29:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741883_1061 (size=93) 2024-11-20T04:29:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741883_1061 (size=93) 2024-11-20T04:29:46,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:46,257 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732076986156) 2024-11-20T04:29:46,274 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 409) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:42657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_539786857_22 at /127.0.0.1:39356 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37089 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:37089 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_539786857_22 at /127.0.0.1:51830 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_539786857_22 at /127.0.0.1:50408 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:42657 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1025 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=6830 (was 6875) 2024-11-20T04:29:46,276 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-20T04:29:46,293 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6822 2024-11-20T04:29:46,293 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-20T04:29:46,313 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:46,318 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T04:29:46,322 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c2a32e16c274,37809,1732076970521 2024-11-20T04:29:46,325 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a89f563 2024-11-20T04:29:46,326 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T04:29:46,328 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T04:29:46,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T04:29:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T04:29:46,340 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T04:29:46,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-20T04:29:46,343 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T04:29:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T04:29:46,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741885_1063 (size=694) 2024-11-20T04:29:46,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741885_1063 (size=694) 2024-11-20T04:29:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741885_1063 (size=694) 2024-11-20T04:29:46,358 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e 2024-11-20T04:29:46,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741886_1064 (size=77) 2024-11-20T04:29:46,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741886_1064 (size=77) 2024-11-20T04:29:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741886_1064 (size=77) 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:46,367 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,367 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,367 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076986367Disabling compacts and flushes for region at 1732076986367Disabling writes for close at 1732076986367Writing region close event to WAL at 1732076986367Closed at 1732076986367 2024-11-20T04:29:46,369 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T04:29:46,373 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732076986369"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732076986369"}]},"ts":"1732076986369"} 2024-11-20T04:29:46,377 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T04:29:46,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T04:29:46,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732076986378"}]},"ts":"1732076986378"} 2024-11-20T04:29:46,385 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-20T04:29:46,386 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c2a32e16c274=0} racks are {/default-rack=0} 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T04:29:46,387 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T04:29:46,387 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T04:29:46,387 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T04:29:46,387 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T04:29:46,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN}] 2024-11-20T04:29:46,390 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN 2024-11-20T04:29:46,392 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN; state=OFFLINE, location=c2a32e16c274,36039,1732076971312; forceNewPlan=false, retain=false 2024-11-20T04:29:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T04:29:46,544 INFO [c2a32e16c274:37809 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T04:29:46,545 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPENING, regionLocation=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:46,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN because future has completed 2024-11-20T04:29:46,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312}] 2024-11-20T04:29:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T04:29:46,703 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T04:29:46,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T04:29:46,710 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,710 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:46,711 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,711 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:46,711 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,711 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,713 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,715 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:46,715 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,715 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:46,715 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,717 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:46,717 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:46,717 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:46,717 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,718 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,718 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,719 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,719 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,720 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T04:29:46,721 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,723 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:46,724 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 46816b69831c7c3d004fa2452f463d5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61916648, jitterRate=-0.0773700475692749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T04:29:46,724 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:46,725 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 46816b69831c7c3d004fa2452f463d5c: Running coprocessor pre-open hook at 1732076986711Writing region info on filesystem at 1732076986711Initializing all the Stores at 1732076986713 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076986713Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076986713Cleaning up temporary data from old regions at 1732076986719 (+6 ms)Running coprocessor post-open hooks at 1732076986724 (+5 ms)Region opened successfully at 1732076986725 (+1 ms) 2024-11-20T04:29:46,726 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., pid=6, masterSystemTime=1732076986703 2024-11-20T04:29:46,729 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,729 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:46,730 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPEN, openSeqNum=2, regionLocation=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:46,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 because future has completed 2024-11-20T04:29:46,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T04:29:46,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 in 186 msec 2024-11-20T04:29:46,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T04:29:46,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN in 350 msec 2024-11-20T04:29:46,744 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T04:29:46,744 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732076986744"}]},"ts":"1732076986744"} 2024-11-20T04:29:46,747 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-20T04:29:46,748 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T04:29:46,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 415 msec 2024-11-20T04:29:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T04:29:46,981 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-20T04:29:46,981 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-20T04:29:46,982 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T04:29:46,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-20T04:29:46,988 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T04:29:46,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-20T04:29:47,001 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=2] 2024-11-20T04:29:47,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T04:29:47,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T04:29:47,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=46816b69831c7c3d004fa2452f463d5c, source=c2a32e16c274,36039,1732076971312, destination=c2a32e16c274,38897,1732076971421, warming up region on c2a32e16c274,38897,1732076971421 2024-11-20T04:29:47,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=46816b69831c7c3d004fa2452f463d5c, source=c2a32e16c274,36039,1732076971312, destination=c2a32e16c274,38897,1732076971421, running balancer 2024-11-20T04:29:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE 2024-11-20T04:29:47,024 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE 2024-11-20T04:29:47,026 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=CLOSING, regionLocation=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:47,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(7855): Warmup {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:47,028 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE because future has completed 2024-11-20T04:29:47,029 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:47,029 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,029 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T04:29:47,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312}] 2024-11-20T04:29:47,030 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:47,030 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,031 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:47,031 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,031 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:47,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076987032Disabling compacts and flushes for region at 1732076987032Disabling writes for close at 1732076987032Writing region close event to WAL at 1732076987033 (+1 ms)Closed at 1732076987033 2024-11-20T04:29:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T04:29:47,190 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,191 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T04:29:47,191 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:47,191 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,191 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,191 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:47,191 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,192 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 46816b69831c7c3d004fa2452f463d5c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-20T04:29:47,209 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/124a3310b5b0455f95b5e5a8adb895f4 is 35, key is r1/cf1:q/1732076987005/Put/seqid=0 2024-11-20T04:29:47,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741887_1065 (size=4783) 2024-11-20T04:29:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741887_1065 (size=4783) 2024-11-20T04:29:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741887_1065 (size=4783) 2024-11-20T04:29:47,217 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/124a3310b5b0455f95b5e5a8adb895f4 2024-11-20T04:29:47,226 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/124a3310b5b0455f95b5e5a8adb895f4 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4 2024-11-20T04:29:47,232 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4, entries=1, sequenceid=5, filesize=4.7 K 2024-11-20T04:29:47,234 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 46816b69831c7c3d004fa2452f463d5c in 41ms, sequenceid=5, compaction requested=false 2024-11-20T04:29:47,234 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-20T04:29:47,240 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T04:29:47,243 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,243 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076987191Running coprocessor pre-close hooks at 1732076987191Disabling compacts and flushes for region at 1732076987191Disabling writes for close at 1732076987191Obtaining lock to block concurrent updates at 1732076987192 (+1 ms)Preparing flush snapshotting stores in 46816b69831c7c3d004fa2452f463d5c at 1732076987192Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732076987192Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. at 1732076987193 (+1 ms)Flushing 46816b69831c7c3d004fa2452f463d5c/cf1: creating writer at 1732076987193Flushing 46816b69831c7c3d004fa2452f463d5c/cf1: appending metadata at 1732076987209 (+16 ms)Flushing 46816b69831c7c3d004fa2452f463d5c/cf1: closing flushed file at 1732076987209Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc5395f: reopening flushed file at 1732076987225 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 46816b69831c7c3d004fa2452f463d5c in 41ms, sequenceid=5, compaction requested=false at 1732076987234 (+9 ms)Writing region close event to WAL at 1732076987235 (+1 ms)Running coprocessor post-close hooks at 1732076987240 (+5 ms)Closed at 1732076987243 (+3 ms) 2024-11-20T04:29:47,243 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 46816b69831c7c3d004fa2452f463d5c move to c2a32e16c274,38897,1732076971421 record at close sequenceid=5 2024-11-20T04:29:47,247 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,248 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=CLOSED 2024-11-20T04:29:47,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 because future has completed 2024-11-20T04:29:47,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T04:29:47,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 in 223 msec 2024-11-20T04:29:47,261 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE; state=CLOSED, location=c2a32e16c274,38897,1732076971421; forceNewPlan=false, retain=false 2024-11-20T04:29:47,412 INFO [c2a32e16c274:37809 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T04:29:47,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPENING, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:47,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE because future has completed 2024-11-20T04:29:47,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421}] 2024-11-20T04:29:47,574 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,575 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:47,575 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,575 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:47,575 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,575 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,577 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,578 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:47,578 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,586 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4 2024-11-20T04:29:47,586 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:47,586 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,587 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:47,587 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,588 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:47,588 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,589 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,590 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,591 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,591 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,592 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T04:29:47,593 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,594 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 46816b69831c7c3d004fa2452f463d5c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67633461, jitterRate=0.007817104458808899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T04:29:47,594 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,595 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 46816b69831c7c3d004fa2452f463d5c: Running coprocessor pre-open hook at 1732076987575Writing region info on filesystem at 1732076987575Initializing all the Stores at 1732076987577 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076987577Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076987577Cleaning up temporary data from old regions at 1732076987591 (+14 ms)Running coprocessor post-open hooks at 1732076987594 (+3 ms)Region opened successfully at 1732076987595 (+1 ms) 2024-11-20T04:29:47,596 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., pid=9, masterSystemTime=1732076987569 2024-11-20T04:29:47,599 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,599 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPEN, openSeqNum=9, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:47,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 because future has completed 2024-11-20T04:29:47,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T04:29:47,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 in 189 msec 2024-11-20T04:29:47,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE in 585 msec 2024-11-20T04:29:47,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T04:29:47,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T04:29:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:47610 deadline: 1732077047637, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=38897 startCode=1732076971421. As of locationSeqNum=5. 2024-11-20T04:29:47,662 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=38897 startCode=1732076971421. As of locationSeqNum=5. 2024-11-20T04:29:47,662 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=38897 startCode=1732076971421. As of locationSeqNum=5. 2024-11-20T04:29:47,662 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=38897 startCode=1732076971421. As of locationSeqNum=5. 2024-11-20T04:29:47,780 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 46816b69831c7c3d004fa2452f463d5c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-20T04:29:47,797 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/0e7760553f014b29bb2f9c613f87c2f0 is 29, key is r1/cf1:/1732076987771/DeleteFamily/seqid=0 2024-11-20T04:29:47,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741888_1066 (size=4906) 2024-11-20T04:29:47,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741888_1066 (size=4906) 2024-11-20T04:29:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741888_1066 (size=4906) 2024-11-20T04:29:47,806 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,813 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,836 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf2/78f56ed323204f08bb7b62d0d0022509 is 29, key is r1/cf2:/1732076987771/DeleteFamily/seqid=0 2024-11-20T04:29:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741889_1067 (size=4906) 2024-11-20T04:29:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741889_1067 (size=4906) 2024-11-20T04:29:47,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741889_1067 (size=4906) 2024-11-20T04:29:47,844 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf2/78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:47,851 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:47,852 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/0e7760553f014b29bb2f9c613f87c2f0 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,859 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,859 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T04:29:47,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf2/78f56ed323204f08bb7b62d0d0022509 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:47,866 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:47,866 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T04:29:47,867 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 46816b69831c7c3d004fa2452f463d5c in 88ms, sequenceid=12, compaction requested=false 2024-11-20T04:29:47,867 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 46816b69831c7c3d004fa2452f463d5c: 2024-11-20T04:29:47,870 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T04:29:47,871 DEBUG [Time-limited test {}] regionserver.HStore(1541): 46816b69831c7c3d004fa2452f463d5c/cf1 is initiating major compaction (all files) 2024-11-20T04:29:47,871 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T04:29:47,871 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:47,872 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 46816b69831c7c3d004fa2452f463d5c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,872 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4, hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0] into tmpdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp, totalSize=9.5 K 2024-11-20T04:29:47,873 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 124a3310b5b0455f95b5e5a8adb895f4, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732076987005 2024-11-20T04:29:47,873 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0e7760553f014b29bb2f9c613f87c2f0, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T04:29:47,886 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 46816b69831c7c3d004fa2452f463d5c#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T04:29:47,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741890_1068 (size=4626) 2024-11-20T04:29:47,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741890_1068 (size=4626) 2024-11-20T04:29:47,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741890_1068 (size=4626) 2024-11-20T04:29:47,903 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf1/107cd2433b0e4aec87715583a456fa96 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/107cd2433b0e4aec87715583a456fa96 2024-11-20T04:29:47,922 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 46816b69831c7c3d004fa2452f463d5c/cf1 of 46816b69831c7c3d004fa2452f463d5c into 107cd2433b0e4aec87715583a456fa96(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T04:29:47,922 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 46816b69831c7c3d004fa2452f463d5c: 2024-11-20T04:29:47,922 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-20T04:29:47,922 DEBUG [Time-limited test {}] regionserver.HStore(1541): 46816b69831c7c3d004fa2452f463d5c/cf2 is initiating major compaction (all files) 2024-11-20T04:29:47,922 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T04:29:47,922 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T04:29:47,923 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 46816b69831c7c3d004fa2452f463d5c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,923 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509] into tmpdir=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp, totalSize=4.8 K 2024-11-20T04:29:47,924 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 78f56ed323204f08bb7b62d0d0022509, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T04:29:47,930 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 46816b69831c7c3d004fa2452f463d5c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T04:29:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741891_1069 (size=4592) 2024-11-20T04:29:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741891_1069 (size=4592) 2024-11-20T04:29:47,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741891_1069 (size=4592) 2024-11-20T04:29:47,948 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/.tmp/cf2/2fbb87b015ca4e60986065108f6c1c58 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/2fbb87b015ca4e60986065108f6c1c58 2024-11-20T04:29:47,957 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 46816b69831c7c3d004fa2452f463d5c/cf2 of 46816b69831c7c3d004fa2452f463d5c into 2fbb87b015ca4e60986065108f6c1c58(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T04:29:47,957 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 46816b69831c7c3d004fa2452f463d5c: 2024-11-20T04:29:47,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=46816b69831c7c3d004fa2452f463d5c, source=c2a32e16c274,38897,1732076971421, destination=c2a32e16c274,36039,1732076971312, warming up region on c2a32e16c274,36039,1732076971312 2024-11-20T04:29:47,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=46816b69831c7c3d004fa2452f463d5c, source=c2a32e16c274,38897,1732076971421, destination=c2a32e16c274,36039,1732076971312, running balancer 2024-11-20T04:29:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE 2024-11-20T04:29:47,965 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE 2024-11-20T04:29:47,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(7855): Warmup {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:47,967 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=CLOSING, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:47,968 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,969 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:47,969 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE because future has completed 2024-11-20T04:29:47,971 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T04:29:47,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421}] 2024-11-20T04:29:47,980 INFO [StoreFileOpener-46816b69831c7c3d004fa2452f463d5c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,980 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:47,985 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/107cd2433b0e4aec87715583a456fa96 2024-11-20T04:29:47,991 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4 2024-11-20T04:29:47,991 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:47,991 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:47,992 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:47,992 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:47,999 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/2fbb87b015ca4e60986065108f6c1c58 2024-11-20T04:29:48,003 INFO [StoreFileOpener-46816b69831c7c3d004fa2452f463d5c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:48,004 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:48,004 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:48,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36039 {}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076988004Disabling compacts and flushes for region at 1732076988004Disabling writes for close at 1732076988004Writing region close event to WAL at 1732076988009 (+5 ms)Closed at 1732076988009 2024-11-20T04:29:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-20T04:29:48,126 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,126 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T04:29:48,126 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:48,126 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,126 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,126 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:48,126 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,127 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4, hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0] to archive 2024-11-20T04:29:48,130 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T04:29:48,133 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4 to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/124a3310b5b0455f95b5e5a8adb895f4 2024-11-20T04:29:48,135 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0 to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/0e7760553f014b29bb2f9c613f87c2f0 2024-11-20T04:29:48,149 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509] to archive 2024-11-20T04:29:48,150 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T04:29:48,152 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509 to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/78f56ed323204f08bb7b62d0d0022509 2024-11-20T04:29:48,157 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-20T04:29:48,158 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,158 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076988126Running coprocessor pre-close hooks at 1732076988126Disabling compacts and flushes for region at 1732076988126Disabling writes for close at 1732076988126Writing region close event to WAL at 1732076988153 (+27 ms)Running coprocessor post-close hooks at 1732076988158 (+5 ms)Closed at 1732076988158 2024-11-20T04:29:48,158 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 46816b69831c7c3d004fa2452f463d5c move to c2a32e16c274,36039,1732076971312 record at close sequenceid=12 2024-11-20T04:29:48,160 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=CLOSED 2024-11-20T04:29:48,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 because future has completed 2024-11-20T04:29:48,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T04:29:48,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 in 195 msec 2024-11-20T04:29:48,170 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE; state=CLOSED, location=c2a32e16c274,36039,1732076971312; forceNewPlan=false, retain=false 2024-11-20T04:29:48,320 INFO [c2a32e16c274:37809 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T04:29:48,321 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPENING, regionLocation=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:48,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE because future has completed 2024-11-20T04:29:48,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312}] 2024-11-20T04:29:48,482 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,482 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:48,482 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,483 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:48,483 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,483 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,484 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,485 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:48,485 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:48,491 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/107cd2433b0e4aec87715583a456fa96 2024-11-20T04:29:48,491 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:48,491 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,492 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:48,493 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:48,500 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/2fbb87b015ca4e60986065108f6c1c58 2024-11-20T04:29:48,500 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:48,500 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,501 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,502 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,503 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,503 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,504 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T04:29:48,506 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,507 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 46816b69831c7c3d004fa2452f463d5c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62855846, jitterRate=-0.06337490677833557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T04:29:48,507 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,508 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 46816b69831c7c3d004fa2452f463d5c: Running coprocessor pre-open hook at 1732076988483Writing region info on filesystem at 1732076988483Initializing all the Stores at 1732076988484 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076988484Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076988484Cleaning up temporary data from old regions at 1732076988503 (+19 ms)Running coprocessor post-open hooks at 1732076988507 (+4 ms)Region opened successfully at 1732076988507 2024-11-20T04:29:48,509 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., pid=12, masterSystemTime=1732076988478 2024-11-20T04:29:48,512 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,512 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPEN, openSeqNum=18, regionLocation=c2a32e16c274,36039,1732076971312 2024-11-20T04:29:48,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 because future has completed 2024-11-20T04:29:48,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-20T04:29:48,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,36039,1732076971312 in 192 msec 2024-11-20T04:29:48,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, REOPEN/MOVE in 557 msec 2024-11-20T04:29:48,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T04:29:48,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T04:29:48,571 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c2a32e16c274,36039,1732076971312: testing ***** 2024-11-20T04:29:48,571 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T04:29:48,574 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T04:29:48,576 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T04:29:48,579 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T04:29:48,580 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T04:29:48,591 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 322123680 }, "NonHeapMemoryUsage": { "committed": 171048960, "init": 7667712, "max": -1, "used": 168509944 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c2a32e16c274", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 11, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 8, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 10, "TotalCallTime_98th_percentile": 10, "TotalCallTime_99th_percentile": 10, "TotalCallTime_99.9th_percentile": 10, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c2a32e16c274", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60520", "tag.serverName": "c2a32e16c274,39781,1732076971499", "tag.clusterId": "80f02adf-6f24-435e-89f5-b374cfcfd2bf", "tag.Context": "regionserver", "tag.Hostname": "c2a32e16c274", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732076971499, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T04:29:48,596 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37809 {}] master.MasterRpcServices(700): c2a32e16c274,36039,1732076971312 reported a fatal error: ***** ABORTING region server c2a32e16c274,36039,1732076971312: testing ***** 2024-11-20T04:29:48,599 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2a32e16c274,36039,1732076971312' ***** 2024-11-20T04:29:48,599 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-20T04:29:48,599 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T04:29:48,600 INFO [RS:0;c2a32e16c274:36039 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T04:29:48,600 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T04:29:48,600 INFO [RS:0;c2a32e16c274:36039 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T04:29:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:42348 deadline: 1732077048600, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=36039 startCode=1732076971312. As of locationSeqNum=12. 2024-11-20T04:29:48,600 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(3091): Received CLOSE for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,601 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(956): aborting server c2a32e16c274,36039,1732076971312 2024-11-20T04:29:48,601 INFO [RS:0;c2a32e16c274:36039 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T04:29:48,601 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=36039 startCode=1732076971312. As of locationSeqNum=12. 2024-11-20T04:29:48,601 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=36039 startCode=1732076971312. As of locationSeqNum=12. 2024-11-20T04:29:48,601 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:29:48,601 INFO [RS:0;c2a32e16c274:36039 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2a32e16c274:36039. 2024-11-20T04:29:48,601 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,601 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2a32e16c274 port=36039 startCode=1732076971312. As of locationSeqNum=12. 2024-11-20T04:29:48,601 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,601 DEBUG [RS:0;c2a32e16c274:36039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T04:29:48,601 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:29:48,601 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,602 DEBUG [RS:0;c2a32e16c274:36039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:29:48,602 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T04:29:48,602 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1325): Online Regions={46816b69831c7c3d004fa2452f463d5c=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.} 2024-11-20T04:29:48,603 DEBUG [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1351): Waiting on 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:48,605 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,605 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732076988601Running coprocessor pre-close hooks at 1732076988601Disabling compacts and flushes for region at 1732076988601Disabling writes for close at 1732076988601Writing region close event to WAL at 1732076988604 (+3 ms)Running coprocessor post-close hooks at 1732076988604Closed at 1732076988605 (+1 ms) 2024-11-20T04:29:48,605 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:48,683 INFO [regionserver/c2a32e16c274:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:29:48,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2a32e16c274,36039,1732076971312 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:48,712 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2a32e16c274,36039,1732076971312 aborting 2024-11-20T04:29:48,712 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2a32e16c274,36039,1732076971312 aborting 2024-11-20T04:29:48,712 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=12 from cache 2024-11-20T04:29:48,803 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(976): stopping server c2a32e16c274,36039,1732076971312; all regions closed. 2024-11-20T04:29:48,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741833_1009 (size=1677) 2024-11-20T04:29:48,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741833_1009 (size=1677) 2024-11-20T04:29:48,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741833_1009 (size=1677) 2024-11-20T04:29:48,808 DEBUG [RS:0;c2a32e16c274:36039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] hbase.ChoreService(370): Chore service for: regionserver/c2a32e16c274:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T04:29:48,808 INFO [regionserver/c2a32e16c274:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T04:29:48,808 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T04:29:48,809 INFO [RS:0;c2a32e16c274:36039 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T04:29:48,809 INFO [RS:0;c2a32e16c274:36039 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36039 2024-11-20T04:29:48,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T04:29:48,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2a32e16c274,36039,1732076971312 2024-11-20T04:29:48,818 INFO [RS:0;c2a32e16c274:36039 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T04:29:48,819 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2a32e16c274,36039,1732076971312] 2024-11-20T04:29:48,821 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2a32e16c274,36039,1732076971312 already deleted, retry=false 2024-11-20T04:29:48,822 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c2a32e16c274,36039,1732076971312 on c2a32e16c274,37809,1732076970521 2024-11-20T04:29:48,827 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c2a32e16c274,36039,1732076971312, splitWal=true, meta=false 2024-11-20T04:29:48,829 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for c2a32e16c274,36039,1732076971312 (carryingMeta=false) c2a32e16c274,36039,1732076971312/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6626fed6[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T04:29:48,829 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c2a32e16c274,36039,1732076971312, splitWal=true, meta=false 2024-11-20T04:29:48,831 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): c2a32e16c274,36039,1732076971312 had 1 regions 2024-11-20T04:29:48,831 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c2a32e16c274,36039,1732076971312, splitWal=true, meta=false, isMeta: false 2024-11-20T04:29:48,833 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting 2024-11-20T04:29:48,834 INFO [PEWorker-1 {}] master.SplitWALManager(105): c2a32e16c274,36039,1732076971312 WAL count=1, meta=false 2024-11-20T04:29:48,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c2a32e16c274%2C36039%2C1732076971312.1732076973120}] 2024-11-20T04:29:48,842 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=c2a32e16c274,39781,1732076971499 2024-11-20T04:29:48,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c2a32e16c274%2C36039%2C1732076971312.1732076973120, worker=c2a32e16c274,39781,1732076971499}] 2024-11-20T04:29:48,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:29:48,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36039-0x10133a3095f0001, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:29:48,922 INFO [RS:0;c2a32e16c274:36039 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T04:29:48,922 INFO [RS:0;c2a32e16c274:36039 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2a32e16c274,36039,1732076971312; zookeeper connection closed. 2024-11-20T04:29:48,922 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4e96939a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4e96939a 2024-11-20T04:29:48,922 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18] 2024-11-20T04:29:48,924 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c2a32e16c274:36039 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2a32e16c274/172.17.0.2:36039 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:48,924 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18, error=java.net.ConnectException: Call to address=c2a32e16c274:36039 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2a32e16c274/172.17.0.2:36039 2024-11-20T04:29:48,924 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-20T04:29:48,925 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 from cache 2024-11-20T04:29:48,925 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address c2a32e16c274:36039 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2a32e16c274/172.17.0.2:36039 2024-11-20T04:29:49,007 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T04:29:49,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46661, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T04:29:49,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39781 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-20T04:29:49,029 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120, size=1.6 K (1677bytes) 2024-11-20T04:29:49,029 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 2024-11-20T04:29:49,030 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 after 1ms 2024-11-20T04:29:49,033 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:49,033 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 took 4ms 2024-11-20T04:29:49,040 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 46816b69831c7c3d004fa2452f463d5c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-20T04:29:49,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 so closing down 2024-11-20T04:29:49,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:49,041 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:49,041 INFO [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-20T04:29:49,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120, journal: Splitting hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120, size=1.6 K (1677bytes) at 1732076989029Finishing writing output for hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 so closing down at 1732076989041 (+12 ms)3 split writer threads finished at 1732076989041Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732076989041 2024-11-20T04:29:49,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 2024-11-20T04:29:49,043 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2a32e16c274:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-20T04:29:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37809 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-20T04:29:49,050 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting/c2a32e16c274%2C36039%2C1732076971312.1732076973120 to hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs 2024-11-20T04:29:49,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T04:29:49,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c2a32e16c274%2C36039%2C1732076971312.1732076973120, worker=c2a32e16c274,39781,1732076971499 in 205 msec 2024-11-20T04:29:49,055 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=c2a32e16c274,39781,1732076971499 2024-11-20T04:29:49,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T04:29:49,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c2a32e16c274%2C36039%2C1732076971312.1732076973120, worker=c2a32e16c274,39781,1732076971499 in 219 msec 2024-11-20T04:29:49,060 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting dir is empty, no logs to split. 2024-11-20T04:29:49,060 INFO [PEWorker-1 {}] master.SplitWALManager(105): c2a32e16c274,36039,1732076971312 WAL count=0, meta=false 2024-11-20T04:29:49,060 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if c2a32e16c274,36039,1732076971312 WAL splitting is done? wals=0, meta=false 2024-11-20T04:29:49,062 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c2a32e16c274,36039,1732076971312 failed, ignore...File hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/WALs/c2a32e16c274,36039,1732076971312-splitting does not exist. 2024-11-20T04:29:49,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN}] 2024-11-20T04:29:49,065 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN 2024-11-20T04:29:49,066 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T04:29:49,216 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(204): Hosts are {c2a32e16c274=0} racks are {/default-rack=0} 2024-11-20T04:29:49,217 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T04:29:49,217 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T04:29:49,217 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T04:29:49,217 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T04:29:49,217 INFO [c2a32e16c274:37809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T04:29:49,217 INFO [c2a32e16c274:37809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T04:29:49,217 DEBUG [c2a32e16c274:37809 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T04:29:49,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPENING, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:49,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN because future has completed 2024-11-20T04:29:49,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421}] 2024-11-20T04:29:49,231 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18] 2024-11-20T04:29:49,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to c2a32e16c274:36039 this server is in the failed servers list 2024-11-20T04:29:49,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c2a32e16c274:36039 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c2a32e16c274:36039 2024-11-20T04:29:49,232 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c2a32e16c274:36039 2024-11-20T04:29:49,233 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,36039,1732076971312, seqNum=18 from cache 2024-11-20T04:29:49,377 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:49,377 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 46816b69831c7c3d004fa2452f463d5c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:49,377 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,377 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:49,377 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,378 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,379 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,380 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf1 2024-11-20T04:29:49,380 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,386 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf1/107cd2433b0e4aec87715583a456fa96 2024-11-20T04:29:49,387 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,387 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,388 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46816b69831c7c3d004fa2452f463d5c columnFamilyName cf2 2024-11-20T04:29:49,388 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,394 DEBUG [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/cf2/2fbb87b015ca4e60986065108f6c1c58 2024-11-20T04:29:49,394 INFO [StoreOpener-46816b69831c7c3d004fa2452f463d5c-1 {}] regionserver.HStore(327): Store=46816b69831c7c3d004fa2452f463d5c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,394 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,395 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,396 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,397 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,397 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,397 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T04:29:49,399 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,400 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 46816b69831c7c3d004fa2452f463d5c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64973365, jitterRate=-0.031821414828300476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T04:29:49,400 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:29:49,401 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 46816b69831c7c3d004fa2452f463d5c: Running coprocessor pre-open hook at 1732076989378Writing region info on filesystem at 1732076989378Initializing all the Stores at 1732076989378Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989378Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989379 (+1 ms)Cleaning up temporary data from old regions at 1732076989397 (+18 ms)Running coprocessor post-open hooks at 1732076989400 (+3 ms)Region opened successfully at 1732076989401 (+1 ms) 2024-11-20T04:29:49,402 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., pid=17, masterSystemTime=1732076989373 2024-11-20T04:29:49,404 DEBUG [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:49,404 INFO [RS_OPEN_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:29:49,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=46816b69831c7c3d004fa2452f463d5c, regionState=OPEN, openSeqNum=18, regionLocation=c2a32e16c274,38897,1732076971421 2024-11-20T04:29:49,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 because future has completed 2024-11-20T04:29:49,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T04:29:49,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 46816b69831c7c3d004fa2452f463d5c, server=c2a32e16c274,38897,1732076971421 in 189 msec 2024-11-20T04:29:49,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-20T04:29:49,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=46816b69831c7c3d004fa2452f463d5c, ASSIGN in 347 msec 2024-11-20T04:29:49,414 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server c2a32e16c274,36039,1732076971312 after splitting done 2024-11-20T04:29:49,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure c2a32e16c274,36039,1732076971312, splitWal=true, meta=false in 591 msec 2024-11-20T04:29:49,743 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., hostname=c2a32e16c274,38897,1732076971421, seqNum=18] 2024-11-20T04:29:49,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 413), OpenFileDescriptor=1049 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6776 (was 6822) 2024-11-20T04:29:49,761 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-11-20T04:29:49,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1049, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6775 2024-11-20T04:29:49,777 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-11-20T04:29:49,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:49,802 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:49,803 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:49,806 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89067132, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-89067132, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:49,824 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89067132/hregion-89067132.1732076989806, exclude list is [], retry=0 2024-11-20T04:29:49,827 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:49,827 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:49,828 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:49,830 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89067132/hregion-89067132.1732076989806 2024-11-20T04:29:49,830 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:49,831 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e2b9b39beed9e38bd78b5332a44e5cad, NAME => 'testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:49,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741893_1071 (size=67) 2024-11-20T04:29:49,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741893_1071 (size=67) 2024-11-20T04:29:49,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741893_1071 (size=67) 2024-11-20T04:29:49,844 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:49,846 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,847 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName a 2024-11-20T04:29:49,847 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,848 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,848 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,849 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName b 2024-11-20T04:29:49,849 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,850 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,850 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,851 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName c 2024-11-20T04:29:49,851 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,852 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,852 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,854 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,854 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,855 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:49,856 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,858 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:49,858 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2b9b39beed9e38bd78b5332a44e5cad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72954715, jitterRate=0.08710996806621552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2b9b39beed9e38bd78b5332a44e5cad: Writing region info on filesystem at 1732076989844Initializing all the Stores at 1732076989845 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989845Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989845Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989845Cleaning up temporary data from old regions at 1732076989854 (+9 ms)Region opened successfully at 1732076989859 (+5 ms) 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2b9b39beed9e38bd78b5332a44e5cad, disabling compactions & flushes 2024-11-20T04:29:49,859 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. after waiting 0 ms 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:49,859 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2b9b39beed9e38bd78b5332a44e5cad: Waiting for close lock at 1732076989859Disabling compacts and flushes for region at 1732076989859Disabling writes for close at 1732076989859Writing region close event to WAL at 1732076989859Closed at 1732076989859 2024-11-20T04:29:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741892_1070 (size=93) 2024-11-20T04:29:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741892_1070 (size=93) 2024-11-20T04:29:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741892_1070 (size=93) 2024-11-20T04:29:49,864 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:49,864 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89067132:(num 1732076989806) 2024-11-20T04:29:49,864 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:49,866 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:49,879 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, exclude list is [], retry=0 2024-11-20T04:29:49,882 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:49,882 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:49,882 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:49,884 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 2024-11-20T04:29:49,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:49,884 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2b9b39beed9e38bd78b5332a44e5cad, NAME => 'testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:49,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:49,884 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,884 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,886 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,886 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName a 2024-11-20T04:29:49,887 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,887 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,887 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,888 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName b 2024-11-20T04:29:49,888 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,888 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,888 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,889 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName c 2024-11-20T04:29:49,889 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:49,889 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:49,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,893 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:49,894 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:49,895 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2b9b39beed9e38bd78b5332a44e5cad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68452382, jitterRate=0.020019978284835815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:49,896 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2b9b39beed9e38bd78b5332a44e5cad: Writing region info on filesystem at 1732076989885Initializing all the Stores at 1732076989885Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989885Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989886 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076989886Cleaning up temporary data from old regions at 1732076989892 (+6 ms)Region opened successfully at 1732076989896 (+4 ms) 2024-11-20T04:29:49,924 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2b9b39beed9e38bd78b5332a44e5cad 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T04:29:49,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/a/93b202a9a9a04174a21c367c5e2c4079 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732076989896/Put/seqid=0 2024-11-20T04:29:49,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741895_1073 (size=5958) 2024-11-20T04:29:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741895_1073 (size=5958) 2024-11-20T04:29:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741895_1073 (size=5958) 2024-11-20T04:29:49,953 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/a/93b202a9a9a04174a21c367c5e2c4079 2024-11-20T04:29:49,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/8e5b6143d5c6464492aa9610ecedda30 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732076989904/Put/seqid=0 2024-11-20T04:29:49,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741896_1074 (size=5958) 2024-11-20T04:29:49,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741896_1074 (size=5958) 2024-11-20T04:29:49,982 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/8e5b6143d5c6464492aa9610ecedda30 2024-11-20T04:29:49,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741896_1074 (size=5958) 2024-11-20T04:29:50,003 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/c/37b7f81c956f4c8c90f3e06af888197f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732076989912/Put/seqid=0 2024-11-20T04:29:50,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741897_1075 (size=5958) 2024-11-20T04:29:50,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741897_1075 (size=5958) 2024-11-20T04:29:50,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741897_1075 (size=5958) 2024-11-20T04:29:50,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/c/37b7f81c956f4c8c90f3e06af888197f 2024-11-20T04:29:50,022 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/a/93b202a9a9a04174a21c367c5e2c4079 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/a/93b202a9a9a04174a21c367c5e2c4079 2024-11-20T04:29:50,028 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/a/93b202a9a9a04174a21c367c5e2c4079, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T04:29:50,030 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/8e5b6143d5c6464492aa9610ecedda30 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/8e5b6143d5c6464492aa9610ecedda30 2024-11-20T04:29:50,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/8e5b6143d5c6464492aa9610ecedda30, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T04:29:50,038 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/c/37b7f81c956f4c8c90f3e06af888197f as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/c/37b7f81c956f4c8c90f3e06af888197f 2024-11-20T04:29:50,044 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/c/37b7f81c956f4c8c90f3e06af888197f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T04:29:50,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e2b9b39beed9e38bd78b5332a44e5cad in 121ms, sequenceid=33, compaction requested=false 2024-11-20T04:29:50,046 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e2b9b39beed9e38bd78b5332a44e5cad: 2024-11-20T04:29:50,046 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2b9b39beed9e38bd78b5332a44e5cad, disabling compactions & flushes 2024-11-20T04:29:50,046 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:50,046 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:50,046 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. after waiting 0 ms 2024-11-20T04:29:50,046 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:50,047 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. 2024-11-20T04:29:50,048 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2b9b39beed9e38bd78b5332a44e5cad: Waiting for close lock at 1732076990046Disabling compacts and flushes for region at 1732076990046Disabling writes for close at 1732076990046Writing region close event to WAL at 1732076990047 (+1 ms)Closed at 1732076990047 2024-11-20T04:29:50,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741894_1072 (size=2873) 2024-11-20T04:29:50,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741894_1072 (size=2873) 2024-11-20T04:29:50,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741894_1072 (size=2873) 2024-11-20T04:29:50,063 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/8e5b6143d5c6464492aa9610ecedda30 to hdfs://localhost:42657/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/8e5b6143d5c6464492aa9610ecedda30 2024-11-20T04:29:50,088 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, size=2.8 K (2873bytes) 2024-11-20T04:29:50,088 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 2024-11-20T04:29:50,089 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 after 1ms 2024-11-20T04:29:50,091 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:50,091 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 took 3ms 2024-11-20T04:29:50,094 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 so closing down 2024-11-20T04:29:50,094 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:50,095 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732076989866.temp 2024-11-20T04:29:50,096 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp 2024-11-20T04:29:50,097 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741898_1076 (size=2312) 2024-11-20T04:29:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741898_1076 (size=2312) 2024-11-20T04:29:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741898_1076 (size=2312) 2024-11-20T04:29:50,106 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:50,107 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 2024-11-20T04:29:50,107 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-20T04:29:50,108 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, journal: Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, size=2.8 K (2873bytes) at 1732076990088Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 so closing down at 1732076990094 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp at 1732076990096 (+2 ms)3 split writer threads finished at 1732076990097 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732076990106 (+9 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000003-wal.1732076989866.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 at 1732076990107 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732076990108 (+1 ms) 2024-11-20T04:29:50,109 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076989866 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076989866 2024-11-20T04:29:50,110 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 2024-11-20T04:29:50,110 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:50,112 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:50,126 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076990112, exclude list is [], retry=0 2024-11-20T04:29:50,129 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:50,129 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:50,129 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:50,133 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076990112 2024-11-20T04:29:50,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:50,133 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2b9b39beed9e38bd78b5332a44e5cad, NAME => 'testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:50,133 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:50,133 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,133 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,135 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,136 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName a 2024-11-20T04:29:50,136 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,142 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/a/93b202a9a9a04174a21c367c5e2c4079 2024-11-20T04:29:50,142 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,142 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,144 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName b 2024-11-20T04:29:50,144 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,144 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,145 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,145 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b9b39beed9e38bd78b5332a44e5cad columnFamilyName c 2024-11-20T04:29:50,145 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,151 DEBUG [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/c/37b7f81c956f4c8c90f3e06af888197f 2024-11-20T04:29:50,151 INFO [StoreOpener-e2b9b39beed9e38bd78b5332a44e5cad-1 {}] regionserver.HStore(327): Store=e2b9b39beed9e38bd78b5332a44e5cad/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,152 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,154 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 2024-11-20T04:29:50,156 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:50,158 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 2024-11-20T04:29:50,158 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2b9b39beed9e38bd78b5332a44e5cad 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T04:29:50,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/74c91242a7b649bfb88ed6f8262e69d9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732076989904/Put/seqid=0 2024-11-20T04:29:50,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741900_1078 (size=5958) 2024-11-20T04:29:50,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741900_1078 (size=5958) 2024-11-20T04:29:50,181 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/74c91242a7b649bfb88ed6f8262e69d9 2024-11-20T04:29:50,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741900_1078 (size=5958) 2024-11-20T04:29:50,187 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/.tmp/b/74c91242a7b649bfb88ed6f8262e69d9 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/74c91242a7b649bfb88ed6f8262e69d9 2024-11-20T04:29:50,192 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/b/74c91242a7b649bfb88ed6f8262e69d9, entries=10, sequenceid=32, filesize=5.8 K 2024-11-20T04:29:50,192 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e2b9b39beed9e38bd78b5332a44e5cad in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-11-20T04:29:50,193 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/0000000000000000032 2024-11-20T04:29:50,194 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,194 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,195 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:50,196 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2b9b39beed9e38bd78b5332a44e5cad 2024-11-20T04:29:50,198 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/e2b9b39beed9e38bd78b5332a44e5cad/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-20T04:29:50,199 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2b9b39beed9e38bd78b5332a44e5cad; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68015283, jitterRate=0.013506695628166199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:50,199 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2b9b39beed9e38bd78b5332a44e5cad: Writing region info on filesystem at 1732076990134Initializing all the Stores at 1732076990135 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990135Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990135Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990135Obtaining lock to block concurrent updates at 1732076990158 (+23 ms)Preparing flush snapshotting stores in e2b9b39beed9e38bd78b5332a44e5cad at 1732076990158Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732076990158Flushing stores of testReplayEditsWrittenViaHRegion,,1732076989800.e2b9b39beed9e38bd78b5332a44e5cad. at 1732076990158Flushing e2b9b39beed9e38bd78b5332a44e5cad/b: creating writer at 1732076990158Flushing e2b9b39beed9e38bd78b5332a44e5cad/b: appending metadata at 1732076990172 (+14 ms)Flushing e2b9b39beed9e38bd78b5332a44e5cad/b: closing flushed file at 1732076990172Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@611f6a6: reopening flushed file at 1732076990186 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e2b9b39beed9e38bd78b5332a44e5cad in 34ms, sequenceid=32, compaction requested=false; wal=null at 1732076990192 (+6 ms)Cleaning up temporary data from old regions at 1732076990194 (+2 ms)Region opened successfully at 1732076990199 (+5 ms) 2024-11-20T04:29:50,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=415 (was 406) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:39508 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:39356 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:50578 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51830 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:50408 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51980 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1049) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6745 (was 6775) 2024-11-20T04:29:50,226 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-20T04:29:50,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6744 2024-11-20T04:29:50,243 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-20T04:29:50,267 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:50,269 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:50,270 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:50,273 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63986724, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-63986724, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:50,287 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63986724/hregion-63986724.1732076990274, exclude list is [], retry=0 2024-11-20T04:29:50,290 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:50,290 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:50,290 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:50,293 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63986724/hregion-63986724.1732076990274 2024-11-20T04:29:50,294 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:50,294 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d3f1f81065cab1af241cdb4aa01ae7c, NAME => 'testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:50,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741902_1080 (size=68) 2024-11-20T04:29:50,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741902_1080 (size=68) 2024-11-20T04:29:50,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741902_1080 (size=68) 2024-11-20T04:29:50,303 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:50,304 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,305 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName a 2024-11-20T04:29:50,306 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,306 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,306 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,307 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName b 2024-11-20T04:29:50,308 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,308 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,308 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,310 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName c 2024-11-20T04:29:50,310 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,310 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,311 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:50,315 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,317 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:50,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d3f1f81065cab1af241cdb4aa01ae7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70120480, jitterRate=0.0448765754699707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:50,319 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d3f1f81065cab1af241cdb4aa01ae7c: Writing region info on filesystem at 1732076990303Initializing all the Stores at 1732076990304 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990304Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990304Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990304Cleaning up temporary data from old regions at 1732076990313 (+9 ms)Region opened successfully at 1732076990319 (+6 ms) 2024-11-20T04:29:50,319 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d3f1f81065cab1af241cdb4aa01ae7c, disabling compactions & flushes 2024-11-20T04:29:50,319 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:50,319 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:50,319 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. after waiting 0 ms 2024-11-20T04:29:50,319 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:50,320 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:50,320 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d3f1f81065cab1af241cdb4aa01ae7c: Waiting for close lock at 1732076990319Disabling compacts and flushes for region at 1732076990319Disabling writes for close at 1732076990319Writing region close event to WAL at 1732076990320 (+1 ms)Closed at 1732076990320 2024-11-20T04:29:50,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741901_1079 (size=93) 2024-11-20T04:29:50,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741901_1079 (size=93) 2024-11-20T04:29:50,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741901_1079 (size=93) 2024-11-20T04:29:50,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:50,329 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63986724:(num 1732076990274) 2024-11-20T04:29:50,329 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:50,331 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:50,344 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, exclude list is [], retry=0 2024-11-20T04:29:50,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:50,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:50,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:50,349 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 2024-11-20T04:29:50,349 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:50,404 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d3f1f81065cab1af241cdb4aa01ae7c, NAME => 'testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:50,407 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,407 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:50,407 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,407 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,409 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,410 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName a 2024-11-20T04:29:50,410 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,411 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,411 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,412 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName b 2024-11-20T04:29:50,412 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,412 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,413 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,413 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName c 2024-11-20T04:29:50,413 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:50,414 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:50,414 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,415 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,417 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,417 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:50,419 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,419 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d3f1f81065cab1af241cdb4aa01ae7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69605422, jitterRate=0.03720161318778992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:50,419 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:50,420 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d3f1f81065cab1af241cdb4aa01ae7c: Running coprocessor pre-open hook at 1732076990407Writing region info on filesystem at 1732076990407Initializing all the Stores at 1732076990408 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990408Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990409 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076990409Cleaning up temporary data from old regions at 1732076990417 (+8 ms)Running coprocessor post-open hooks at 1732076990420 (+3 ms)Region opened successfully at 1732076990420 2024-11-20T04:29:50,434 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d3f1f81065cab1af241cdb4aa01ae7c 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-20T04:29:50,435 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:50,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-20T04:29:50,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:50,996 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-20T04:29:50,996 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:50,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T04:29:50,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:50,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-20T04:29:50,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:50,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-20T04:29:50,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-20T04:29:51,436 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:52,241 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T04:29:52,437 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:53,437 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:54,438 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:55,439 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:56,439 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:57,440 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:58,441 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:59,441 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 0d3f1f81065cab1af241cdb4aa01ae7c/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:29:59,443 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0d3f1f81065cab1af241cdb4aa01ae7c: 2024-11-20T04:29:59,443 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0d3f1f81065cab1af241cdb4aa01ae7c: 2024-11-20T04:29:59,457 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-20T04:29:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d3f1f81065cab1af241cdb4aa01ae7c, disabling compactions & flushes 2024-11-20T04:29:59,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. after waiting 0 ms 2024-11-20T04:29:59,458 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,458 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,458 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. 2024-11-20T04:29:59,458 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d3f1f81065cab1af241cdb4aa01ae7c: Waiting for close lock at 1732076999457Running coprocessor pre-close hooks at 1732076999457Disabling compacts and flushes for region at 1732076999457Disabling writes for close at 1732076999457Writing region close event to WAL at 1732076999458 (+1 ms)Running coprocessor post-close hooks at 1732076999458Closed at 1732076999458 2024-11-20T04:29:59,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741903_1081 (size=2398) 2024-11-20T04:29:59,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741903_1081 (size=2398) 2024-11-20T04:29:59,461 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 not finished, retry = 0 2024-11-20T04:29:59,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741903_1081 (size=2398) 2024-11-20T04:29:59,579 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, size=2.3 K (2398bytes) 2024-11-20T04:29:59,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 2024-11-20T04:29:59,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 after 0ms 2024-11-20T04:29:59,582 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:59,582 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 took 4ms 2024-11-20T04:29:59,584 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 so closing down 2024-11-20T04:29:59,584 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:29:59,585 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732076990331.temp 2024-11-20T04:29:59,587 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp 2024-11-20T04:29:59,587 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:29:59,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741904_1082 (size=1672) 2024-11-20T04:29:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741904_1082 (size=1672) 2024-11-20T04:29:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741904_1082 (size=1672) 2024-11-20T04:29:59,594 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-20T04:29:59,595 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 2024-11-20T04:29:59,595 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-20T04:29:59,595 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, journal: Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, size=2.3 K (2398bytes) at 1732076999579Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 so closing down at 1732076999584 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp at 1732076999587 (+3 ms)3 split writer threads finished at 1732076999587Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732076999594 (+7 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000004-wal.1732076990331.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 at 1732076999595 (+1 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732076999595 2024-11-20T04:29:59,597 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076990331 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076990331 2024-11-20T04:29:59,598 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 2024-11-20T04:29:59,598 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:59,598 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T04:29:59,599 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:59,613 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076999600, exclude list is [], retry=0 2024-11-20T04:29:59,616 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:59,617 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:59,617 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:59,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076999600 2024-11-20T04:29:59,619 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:29:59,619 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d3f1f81065cab1af241cdb4aa01ae7c, NAME => 'testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:59,620 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,620 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:59,620 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,620 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,622 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,623 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName a 2024-11-20T04:29:59,623 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:59,623 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:59,624 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,624 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName b 2024-11-20T04:29:59,625 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:59,625 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:59,625 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,626 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d3f1f81065cab1af241cdb4aa01ae7c columnFamilyName c 2024-11-20T04:29:59,626 DEBUG [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:59,626 INFO [StoreOpener-0d3f1f81065cab1af241cdb4aa01ae7c-1 {}] regionserver.HStore(327): Store=0d3f1f81065cab1af241cdb4aa01ae7c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,627 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,629 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 2024-11-20T04:29:59,631 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:29:59,633 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 2024-11-20T04:29:59,633 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d3f1f81065cab1af241cdb4aa01ae7c 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-20T04:29:59,648 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/a/f742a734fdf6418db16ed17a45545f99 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732076999448/Put/seqid=0 2024-11-20T04:29:59,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741906_1084 (size=5523) 2024-11-20T04:29:59,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741906_1084 (size=5523) 2024-11-20T04:29:59,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741906_1084 (size=5523) 2024-11-20T04:29:59,656 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/a/f742a734fdf6418db16ed17a45545f99 2024-11-20T04:29:59,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/b/491bfc6c3d294917bc20839ee7d2738c is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732076999443/Put/seqid=0 2024-11-20T04:29:59,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741907_1085 (size=5524) 2024-11-20T04:29:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741907_1085 (size=5524) 2024-11-20T04:29:59,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741907_1085 (size=5524) 2024-11-20T04:29:59,683 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/b/491bfc6c3d294917bc20839ee7d2738c 2024-11-20T04:29:59,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/c/6f96a146e86841079db3dd76dd438d1b is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732076999446/Put/seqid=0 2024-11-20T04:29:59,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741908_1086 (size=5457) 2024-11-20T04:29:59,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741908_1086 (size=5457) 2024-11-20T04:29:59,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741908_1086 (size=5457) 2024-11-20T04:29:59,709 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/c/6f96a146e86841079db3dd76dd438d1b 2024-11-20T04:29:59,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/a/f742a734fdf6418db16ed17a45545f99 as hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/a/f742a734fdf6418db16ed17a45545f99 2024-11-20T04:29:59,722 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/a/f742a734fdf6418db16ed17a45545f99, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T04:29:59,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/b/491bfc6c3d294917bc20839ee7d2738c as hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/b/491bfc6c3d294917bc20839ee7d2738c 2024-11-20T04:29:59,728 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/b/491bfc6c3d294917bc20839ee7d2738c, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T04:29:59,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/.tmp/c/6f96a146e86841079db3dd76dd438d1b as hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/c/6f96a146e86841079db3dd76dd438d1b 2024-11-20T04:29:59,734 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/c/6f96a146e86841079db3dd76dd438d1b, entries=6, sequenceid=26, filesize=5.3 K 2024-11-20T04:29:59,734 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 0d3f1f81065cab1af241cdb4aa01ae7c in 101ms, sequenceid=26, compaction requested=false; wal=null 2024-11-20T04:29:59,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/0000000000000000026 2024-11-20T04:29:59,736 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,736 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,736 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:29:59,738 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,740 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsAfterAbortingFlush/0d3f1f81065cab1af241cdb4aa01ae7c/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-20T04:29:59,741 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d3f1f81065cab1af241cdb4aa01ae7c; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72960381, jitterRate=0.08719439804553986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:29:59,742 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d3f1f81065cab1af241cdb4aa01ae7c 2024-11-20T04:29:59,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d3f1f81065cab1af241cdb4aa01ae7c: Running coprocessor pre-open hook at 1732076999620Writing region info on filesystem at 1732076999620Initializing all the Stores at 1732076999621 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076999621Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076999621Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076999621Obtaining lock to block concurrent updates at 1732076999633 (+12 ms)Preparing flush snapshotting stores in 0d3f1f81065cab1af241cdb4aa01ae7c at 1732076999633Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732076999633Flushing stores of testReplayEditsAfterAbortingFlush,,1732076990268.0d3f1f81065cab1af241cdb4aa01ae7c. at 1732076999633Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/a: creating writer at 1732076999633Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/a: appending metadata at 1732076999648 (+15 ms)Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/a: closing flushed file at 1732076999648Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/b: creating writer at 1732076999661 (+13 ms)Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/b: appending metadata at 1732076999675 (+14 ms)Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/b: closing flushed file at 1732076999675Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/c: creating writer at 1732076999689 (+14 ms)Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/c: appending metadata at 1732076999702 (+13 ms)Flushing 0d3f1f81065cab1af241cdb4aa01ae7c/c: closing flushed file at 1732076999702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8edbf8: reopening flushed file at 1732076999715 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@246cd46a: reopening flushed file at 1732076999722 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16fc16a7: reopening flushed file at 1732076999728 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 0d3f1f81065cab1af241cdb4aa01ae7c in 101ms, sequenceid=26, compaction requested=false; wal=null at 1732076999734 (+6 ms)Cleaning up temporary data from old regions at 1732076999736 (+2 ms)Running coprocessor post-open hooks at 1732076999742 (+6 ms)Region opened successfully at 1732076999742 2024-11-20T04:29:59,764 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=415 (was 415), OpenFileDescriptor=1181 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6744) - AvailableMemoryMB LEAK? - 2024-11-20T04:29:59,764 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-11-20T04:29:59,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1181, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6855 2024-11-20T04:29:59,777 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-11-20T04:29:59,791 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:59,793 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:29:59,794 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:29:59,796 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-14514760, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-14514760, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:59,809 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-14514760/hregion-14514760.1732076999797, exclude list is [], retry=0 2024-11-20T04:29:59,812 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:59,813 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:59,813 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:59,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-14514760/hregion-14514760.1732076999797 2024-11-20T04:29:59,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:29:59,817 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 44a7d1474f0c30309c4e583897bffb07, NAME => 'testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:29:59,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741910_1088 (size=61) 2024-11-20T04:29:59,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741910_1088 (size=61) 2024-11-20T04:29:59,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741910_1088 (size=61) 2024-11-20T04:29:59,827 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:59,828 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,830 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a7d1474f0c30309c4e583897bffb07 columnFamilyName a 2024-11-20T04:29:59,830 DEBUG [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:59,830 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(327): Store=44a7d1474f0c30309c4e583897bffb07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:59,830 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,831 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,832 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,832 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,834 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,837 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:29:59,837 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a7d1474f0c30309c4e583897bffb07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74588141, jitterRate=0.11144991219043732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:59,838 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a7d1474f0c30309c4e583897bffb07: Writing region info on filesystem at 1732076999827Initializing all the Stores at 1732076999828 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076999828Cleaning up temporary data from old regions at 1732076999832 (+4 ms)Region opened successfully at 1732076999838 (+6 ms) 2024-11-20T04:29:59,838 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 44a7d1474f0c30309c4e583897bffb07, disabling compactions & flushes 2024-11-20T04:29:59,838 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,838 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,838 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. after waiting 0 ms 2024-11-20T04:29:59,838 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,843 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,843 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 44a7d1474f0c30309c4e583897bffb07: Waiting for close lock at 1732076999838Disabling compacts and flushes for region at 1732076999838Disabling writes for close at 1732076999838Writing region close event to WAL at 1732076999843 (+5 ms)Closed at 1732076999843 2024-11-20T04:29:59,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741909_1087 (size=93) 2024-11-20T04:29:59,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741909_1087 (size=93) 2024-11-20T04:29:59,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741909_1087 (size=93) 2024-11-20T04:29:59,849 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:29:59,849 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-14514760:(num 1732076999797) 2024-11-20T04:29:59,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:29:59,851 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:29:59,865 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, exclude list is [], retry=0 2024-11-20T04:29:59,867 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:29:59,868 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:29:59,868 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:29:59,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 2024-11-20T04:29:59,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:29:59,870 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 44a7d1474f0c30309c4e583897bffb07, NAME => 'testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:29:59,870 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:29:59,870 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,870 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,871 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,872 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a7d1474f0c30309c4e583897bffb07 columnFamilyName a 2024-11-20T04:29:59,872 DEBUG [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:29:59,873 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(327): Store=44a7d1474f0c30309c4e583897bffb07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:29:59,873 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,875 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,875 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:29:59,877 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a7d1474f0c30309c4e583897bffb07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66097870, jitterRate=-0.015064984560012817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:29:59,878 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a7d1474f0c30309c4e583897bffb07: Writing region info on filesystem at 1732076999870Initializing all the Stores at 1732076999871 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732076999871Cleaning up temporary data from old regions at 1732076999875 (+4 ms)Region opened successfully at 1732076999878 (+3 ms) 2024-11-20T04:29:59,886 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 44a7d1474f0c30309c4e583897bffb07, disabling compactions & flushes 2024-11-20T04:29:59,886 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,886 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,886 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. after waiting 0 ms 2024-11-20T04:29:59,886 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,886 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,886 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. 2024-11-20T04:29:59,886 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 44a7d1474f0c30309c4e583897bffb07: Waiting for close lock at 1732076999886Disabling compacts and flushes for region at 1732076999886Disabling writes for close at 1732076999886Writing region close event to WAL at 1732076999886Closed at 1732076999886 2024-11-20T04:29:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741911_1089 (size=838) 2024-11-20T04:29:59,889 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 not finished, retry = 0 2024-11-20T04:29:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741911_1089 (size=838) 2024-11-20T04:29:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741911_1089 (size=838) 2024-11-20T04:30:00,006 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, size=838 (838bytes) 2024-11-20T04:30:00,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 2024-11-20T04:30:00,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 after 0ms 2024-11-20T04:30:00,008 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:00,009 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 took 4ms 2024-11-20T04:30:00,010 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 so closing down 2024-11-20T04:30:00,010 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:30:00,012 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732076999851.temp 2024-11-20T04:30:00,013 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp 2024-11-20T04:30:00,013 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:30:00,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741912_1090 (size=838) 2024-11-20T04:30:00,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741912_1090 (size=838) 2024-11-20T04:30:00,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741912_1090 (size=838) 2024-11-20T04:30:00,020 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-20T04:30:00,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp to hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 2024-11-20T04:30:00,022 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, size=838, length=838, corrupted=false, cancelled=false 2024-11-20T04:30:00,022 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, journal: Splitting hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, size=838 (838bytes) at 1732077000006Finishing writing output for hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 so closing down at 1732077000010 (+4 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp at 1732077000013 (+3 ms)3 split writer threads finished at 1732077000013Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732077000020 (+7 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000003-wal.1732076999851.temp to hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 at 1732077000022 (+2 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851, size=838, length=838, corrupted=false, cancelled=false at 1732077000022 2024-11-20T04:30:00,023 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732076999851 to hdfs://localhost:42657/hbase/oldWALs/wal.1732076999851 2024-11-20T04:30:00,024 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 2024-11-20T04:30:00,028 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:00,382 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:00,384 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:00,397 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732077000384, exclude list is [], retry=0 2024-11-20T04:30:00,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:00,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:00,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:00,402 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732077000384 2024-11-20T04:30:00,402 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:30:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 44a7d1474f0c30309c4e583897bffb07, NAME => 'testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,403 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,406 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,408 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a7d1474f0c30309c4e583897bffb07 columnFamilyName a 2024-11-20T04:30:00,408 DEBUG [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:00,409 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(327): Store=44a7d1474f0c30309c4e583897bffb07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:00,409 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,411 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 2024-11-20T04:30:00,413 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:00,414 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 2024-11-20T04:30:00,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 44a7d1474f0c30309c4e583897bffb07 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-20T04:30:00,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/.tmp/a/89e3ac396b9f43648fda478877a00375 is 79, key is testDatalossWhenInputError/a:x0/1732076999878/Put/seqid=0 2024-11-20T04:30:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741914_1092 (size=5808) 2024-11-20T04:30:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741914_1092 (size=5808) 2024-11-20T04:30:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741914_1092 (size=5808) 2024-11-20T04:30:00,443 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/.tmp/a/89e3ac396b9f43648fda478877a00375 2024-11-20T04:30:00,453 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/.tmp/a/89e3ac396b9f43648fda478877a00375 as hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/a/89e3ac396b9f43648fda478877a00375 2024-11-20T04:30:00,460 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/a/89e3ac396b9f43648fda478877a00375, entries=10, sequenceid=12, filesize=5.7 K 2024-11-20T04:30:00,460 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 44a7d1474f0c30309c4e583897bffb07 in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-11-20T04:30:00,461 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/0000000000000000012 2024-11-20T04:30:00,462 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,462 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,465 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,467 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-20T04:30:00,467 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a7d1474f0c30309c4e583897bffb07; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60771137, jitterRate=-0.09443949162960052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:30:00,468 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a7d1474f0c30309c4e583897bffb07: Writing region info on filesystem at 1732077000403Initializing all the Stores at 1732077000406 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077000406Obtaining lock to block concurrent updates at 1732077000414 (+8 ms)Preparing flush snapshotting stores in 44a7d1474f0c30309c4e583897bffb07 at 1732077000414Finished memstore snapshotting testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732077000414Flushing stores of testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07. at 1732077000414Flushing 44a7d1474f0c30309c4e583897bffb07/a: creating writer at 1732077000414Flushing 44a7d1474f0c30309c4e583897bffb07/a: appending metadata at 1732077000432 (+18 ms)Flushing 44a7d1474f0c30309c4e583897bffb07/a: closing flushed file at 1732077000432Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33842ae3: reopening flushed file at 1732077000451 (+19 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 44a7d1474f0c30309c4e583897bffb07 in 46ms, sequenceid=12, compaction requested=false; wal=null at 1732077000460 (+9 ms)Cleaning up temporary data from old regions at 1732077000462 (+2 ms)Region opened successfully at 1732077000468 (+6 ms) 2024-11-20T04:30:00,471 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 44a7d1474f0c30309c4e583897bffb07, NAME => 'testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:00,471 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732076999792.44a7d1474f0c30309c4e583897bffb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:00,472 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,472 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,473 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,474 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a7d1474f0c30309c4e583897bffb07 columnFamilyName a 2024-11-20T04:30:00,474 DEBUG [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:00,479 DEBUG [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/a/89e3ac396b9f43648fda478877a00375 2024-11-20T04:30:00,479 INFO [StoreOpener-44a7d1474f0c30309c4e583897bffb07-1 {}] regionserver.HStore(327): Store=44a7d1474f0c30309c4e583897bffb07/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:00,479 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,481 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,481 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,481 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,483 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a7d1474f0c30309c4e583897bffb07 2024-11-20T04:30:00,485 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testDatalossWhenInputError/44a7d1474f0c30309c4e583897bffb07/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-20T04:30:00,486 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a7d1474f0c30309c4e583897bffb07; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58871508, jitterRate=-0.12274616956710815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T04:30:00,486 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a7d1474f0c30309c4e583897bffb07: Writing region info on filesystem at 1732077000472Initializing all the Stores at 1732077000472Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077000472Cleaning up temporary data from old regions at 1732077000481 (+9 ms)Region opened successfully at 1732077000486 (+5 ms) 2024-11-20T04:30:00,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:53998 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46648 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:53924 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51920 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:51814 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:46578 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2101640850-172.17.0.2-1732076966903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1181) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6843 (was 6855) 2024-11-20T04:30:00,504 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-20T04:30:00,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6842 2024-11-20T04:30:00,517 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-20T04:30:00,532 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:30:00,534 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:30:00,534 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:30:00,536 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58715490, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-58715490, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:00,550 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58715490/hregion-58715490.1732077000537, exclude list is [], retry=0 2024-11-20T04:30:00,553 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:00,553 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:00,554 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:00,556 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58715490/hregion-58715490.1732077000537 2024-11-20T04:30:00,556 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:30:00,556 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9b0f617f775437b9c84fea00923e6d66, NAME => 'testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:30:00,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741916_1094 (size=63) 2024-11-20T04:30:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741916_1094 (size=63) 2024-11-20T04:30:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741916_1094 (size=63) 2024-11-20T04:30:00,969 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:00,971 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,972 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName a 2024-11-20T04:30:00,973 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:00,973 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:00,973 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,974 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName b 2024-11-20T04:30:00,974 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:00,975 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:00,975 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,976 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName c 2024-11-20T04:30:00,976 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:00,977 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:00,977 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,977 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,979 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,979 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,979 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:00,981 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:00,983 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:30:00,983 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b0f617f775437b9c84fea00923e6d66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69396740, jitterRate=0.0340920090675354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b0f617f775437b9c84fea00923e6d66: Writing region info on filesystem at 1732077000970Initializing all the Stores at 1732077000971 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077000971Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077000971Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077000971Cleaning up temporary data from old regions at 1732077000979 (+8 ms)Region opened successfully at 1732077000984 (+5 ms) 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9b0f617f775437b9c84fea00923e6d66, disabling compactions & flushes 2024-11-20T04:30:00,984 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. after waiting 0 ms 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:00,984 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:00,984 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9b0f617f775437b9c84fea00923e6d66: Waiting for close lock at 1732077000984Disabling compacts and flushes for region at 1732077000984Disabling writes for close at 1732077000984Writing region close event to WAL at 1732077000984Closed at 1732077000984 2024-11-20T04:30:00,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741915_1093 (size=93) 2024-11-20T04:30:00,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741915_1093 (size=93) 2024-11-20T04:30:00,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741915_1093 (size=93) 2024-11-20T04:30:00,989 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:30:00,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58715490:(num 1732077000537) 2024-11-20T04:30:00,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:00,991 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:00,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-20T04:30:00,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-20T04:30:01,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, exclude list is [], retry=0 2024-11-20T04:30:01,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:01,009 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:01,009 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:01,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 2024-11-20T04:30:01,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:30:01,011 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b0f617f775437b9c84fea00923e6d66, NAME => 'testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:01,011 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:01,011 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,011 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,013 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,014 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName a 2024-11-20T04:30:01,014 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:01,014 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:01,014 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,015 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName b 2024-11-20T04:30:01,015 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:01,016 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:01,016 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,016 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName c 2024-11-20T04:30:01,016 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:01,017 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:01,017 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,018 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,020 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,020 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:01,022 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:01,022 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b0f617f775437b9c84fea00923e6d66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58884091, jitterRate=-0.12255866825580597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:01,023 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b0f617f775437b9c84fea00923e6d66: Writing region info on filesystem at 1732077001011Initializing all the Stores at 1732077001012 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077001012Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077001013 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077001013Cleaning up temporary data from old regions at 1732077001020 (+7 ms)Region opened successfully at 1732077001023 (+3 ms) 2024-11-20T04:30:01,027 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732077001027/Put/seqid=0 2024-11-20T04:30:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741918_1096 (size=4875) 2024-11-20T04:30:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741918_1096 (size=4875) 2024-11-20T04:30:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741918_1096 (size=4875) 2024-11-20T04:30:01,038 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732077001038/Put/seqid=0 2024-11-20T04:30:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741919_1097 (size=4875) 2024-11-20T04:30:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741919_1097 (size=4875) 2024-11-20T04:30:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741919_1097 (size=4875) 2024-11-20T04:30:01,050 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732077001050/Put/seqid=0 2024-11-20T04:30:01,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741920_1098 (size=4875) 2024-11-20T04:30:01,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741920_1098 (size=4875) 2024-11-20T04:30:01,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741920_1098 (size=4875) 2024-11-20T04:30:01,060 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,064 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-20T04:30:01,064 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T04:30:01,064 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,067 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-20T04:30:01,067 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T04:30:01,067 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,070 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-20T04:30:01,070 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T04:30:01,071 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9b0f617f775437b9c84fea00923e6d66 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-20T04:30:01,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp/a/4b2ed62cfd154e1a8bcf8642249381c8 is 55, key is testCompactedBulkLoadedFiles/a:a/1732077001023/Put/seqid=0 2024-11-20T04:30:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741921_1099 (size=5107) 2024-11-20T04:30:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741921_1099 (size=5107) 2024-11-20T04:30:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741921_1099 (size=5107) 2024-11-20T04:30:01,094 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp/a/4b2ed62cfd154e1a8bcf8642249381c8 2024-11-20T04:30:01,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp/a/4b2ed62cfd154e1a8bcf8642249381c8 as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8 2024-11-20T04:30:01,104 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8, entries=1, sequenceid=4, filesize=5.0 K 2024-11-20T04:30:01,105 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 9b0f617f775437b9c84fea00923e6d66 in 34ms, sequenceid=4, compaction requested=false 2024-11-20T04:30:01,105 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9b0f617f775437b9c84fea00923e6d66: 2024-11-20T04:30:01,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ 2024-11-20T04:30:01,107 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ 2024-11-20T04:30:01,108 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ 2024-11-20T04:30:01,108 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile0 into 9b0f617f775437b9c84fea00923e6d66/a as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ - updating store file list. 2024-11-20T04:30:01,113 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:01,113 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ into 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,113 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile0 into 9b0f617f775437b9c84fea00923e6d66/a (new location: hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_) 2024-11-20T04:30:01,114 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile1 into 9b0f617f775437b9c84fea00923e6d66/a as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ - updating store file list. 2024-11-20T04:30:01,118 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:01,118 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ into 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,119 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile1 into 9b0f617f775437b9c84fea00923e6d66/a (new location: hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_) 2024-11-20T04:30:01,120 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile2 into 9b0f617f775437b9c84fea00923e6d66/a as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ - updating store file list. 2024-11-20T04:30:01,123 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:01,124 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ into 9b0f617f775437b9c84fea00923e6d66/a 2024-11-20T04:30:01,124 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42657/hbase/testCompactedBulkLoadedFiles/hfile2 into 9b0f617f775437b9c84fea00923e6d66/a (new location: hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_) 2024-11-20T04:30:01,132 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T04:30:01,132 DEBUG [Time-limited test {}] regionserver.HStore(1541): 9b0f617f775437b9c84fea00923e6d66/a is initiating major compaction (all files) 2024-11-20T04:30:01,132 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 9b0f617f775437b9c84fea00923e6d66/a in testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:01,132 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_] into tmpdir=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp, totalSize=19.3 K 2024-11-20T04:30:01,133 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4b2ed62cfd154e1a8bcf8642249381c8, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732077001023 2024-11-20T04:30:01,133 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T04:30:01,133 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T04:30:01,134 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T04:30:01,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp/a/b33b72ef442a4124b651c78f0112cd36 is 55, key is testCompactedBulkLoadedFiles/a:a/1732077001023/Put/seqid=0 2024-11-20T04:30:01,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741922_1100 (size=6154) 2024-11-20T04:30:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741922_1100 (size=6154) 2024-11-20T04:30:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741922_1100 (size=6154) 2024-11-20T04:30:01,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/.tmp/a/b33b72ef442a4124b651c78f0112cd36 as hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/b33b72ef442a4124b651c78f0112cd36 2024-11-20T04:30:01,166 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 9b0f617f775437b9c84fea00923e6d66/a of 9b0f617f775437b9c84fea00923e6d66 into b33b72ef442a4124b651c78f0112cd36(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T04:30:01,166 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 9b0f617f775437b9c84fea00923e6d66: 2024-11-20T04:30:01,166 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T04:30:01,166 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T04:30:01,199 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, size=0 (0bytes) 2024-11-20T04:30:01,199 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 might be still open, length is 0 2024-11-20T04:30:01,199 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 2024-11-20T04:30:01,199 WARN [IPC Server handler 1 on default port 42657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-20T04:30:01,200 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 after 1ms 2024-11-20T04:30:02,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56456 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46091:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56456 dst: /127.0.0.1:46091 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46091 remote=/127.0.0.1:56456]. Total timeout mills is 60000, 58777 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:02,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:42374 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42374 dst: /127.0.0.1:43305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:02,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:38138 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45985:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38138 dst: /127.0.0.1:45985 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:02,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741917_1101 (size=1368) 2024-11-20T04:30:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741917_1101 (size=1368) 2024-11-20T04:30:04,981 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T04:30:05,200 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 after 4001ms 2024-11-20T04:30:05,203 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:05,204 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 took 4006ms 2024-11-20T04:30:05,205 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992; continuing. 2024-11-20T04:30:05,205 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 so closing down 2024-11-20T04:30:05,205 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:30:05,207 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732077000992.temp 2024-11-20T04:30:05,208 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp 2024-11-20T04:30:05,208 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:30:05,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741923_1102 (size=635) 2024-11-20T04:30:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741923_1102 (size=635) 2024-11-20T04:30:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741923_1102 (size=635) 2024-11-20T04:30:05,215 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T04:30:05,217 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp to hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 2024-11-20T04:30:05,217 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T04:30:05,217 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, journal: Splitting hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, size=0 (0bytes) at 1732077001199Finishing writing output for hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 so closing down at 1732077005205 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp at 1732077005208 (+3 ms)3 split writer threads finished at 1732077005208Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732077005216 (+8 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000003-wal.1732077000992.temp to hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 at 1732077005217 (+1 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992, size=0, length=0, corrupted=false, cancelled=false at 1732077005217 2024-11-20T04:30:05,219 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 to hdfs://localhost:42657/hbase/oldWALs/wal.1732077000992 2024-11-20T04:30:05,220 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 2024-11-20T04:30:05,220 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:05,222 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:05,236 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077005222, exclude list is [], retry=0 2024-11-20T04:30:05,238 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:05,239 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:05,239 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:05,240 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077005222 2024-11-20T04:30:05,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:30:05,241 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b0f617f775437b9c84fea00923e6d66, NAME => 'testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:05,241 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:05,241 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,241 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,242 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,243 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName a 2024-11-20T04:30:05,243 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,249 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8 2024-11-20T04:30:05,253 DEBUG [StoreFileOpener-9b0f617f775437b9c84fea00923e6d66-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:05,253 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ 2024-11-20T04:30:05,256 DEBUG [StoreFileOpener-9b0f617f775437b9c84fea00923e6d66-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:05,256 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ 2024-11-20T04:30:05,260 DEBUG [StoreFileOpener-9b0f617f775437b9c84fea00923e6d66-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T04:30:05,260 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ 2024-11-20T04:30:05,263 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/b33b72ef442a4124b651c78f0112cd36 2024-11-20T04:30:05,264 WARN [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-20T04:30:05,264 WARN [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-20T04:30:05,264 WARN [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-20T04:30:05,264 WARN [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-20T04:30:05,264 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_] to archive 2024-11-20T04:30:05,265 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T04:30:05,266 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8 to hdfs://localhost:42657/hbase/archive/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/4b2ed62cfd154e1a8bcf8642249381c8 2024-11-20T04:30:05,267 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ to hdfs://localhost:42657/hbase/archive/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_ 2024-11-20T04:30:05,268 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ to hdfs://localhost:42657/hbase/archive/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_ 2024-11-20T04:30:05,269 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ to hdfs://localhost:42657/hbase/archive/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/a/ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_ 2024-11-20T04:30:05,269 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,269 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,270 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName b 2024-11-20T04:30:05,270 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,270 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,270 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,271 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0f617f775437b9c84fea00923e6d66 columnFamilyName c 2024-11-20T04:30:05,271 DEBUG [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,271 INFO [StoreOpener-9b0f617f775437b9c84fea00923e6d66-1 {}] regionserver.HStore(327): Store=9b0f617f775437b9c84fea00923e6d66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,271 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,274 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 2024-11-20T04:30:05,275 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:05,277 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 9b0f617f775437b9c84fea00923e6d66 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "9b0f617f775437b9c84fea00923e6d66" family_name: "a" compaction_input: "4b2ed62cfd154e1a8bcf8642249381c8" compaction_input: "ab31487bdbf14665b4cda05a31b5b7d5_SeqId_4_" compaction_input: "94e7e8bea1b74765b187e47d6069ff2c_SeqId_4_" compaction_input: "732bffbf14274cf0a9533b0fa7f1c455_SeqId_4_" compaction_output: "b33b72ef442a4124b651c78f0112cd36" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-20T04:30:05,277 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-20T04:30:05,277 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 2024-11-20T04:30:05,278 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/0000000000000000008 2024-11-20T04:30:05,278 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,278 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,279 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:05,280 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b0f617f775437b9c84fea00923e6d66 2024-11-20T04:30:05,283 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testCompactedBulkLoadedFiles/9b0f617f775437b9c84fea00923e6d66/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T04:30:05,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b0f617f775437b9c84fea00923e6d66; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63173482, jitterRate=-0.05864176154136658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:05,284 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b0f617f775437b9c84fea00923e6d66: Writing region info on filesystem at 1732077005241Initializing all the Stores at 1732077005242 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005242Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005242Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005242Cleaning up temporary data from old regions at 1732077005278 (+36 ms)Region opened successfully at 1732077005284 (+6 ms) 2024-11-20T04:30:05,286 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9b0f617f775437b9c84fea00923e6d66, disabling compactions & flushes 2024-11-20T04:30:05,286 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:05,286 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:05,286 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. after waiting 0 ms 2024-11-20T04:30:05,287 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:05,287 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732077000532.9b0f617f775437b9c84fea00923e6d66. 2024-11-20T04:30:05,287 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9b0f617f775437b9c84fea00923e6d66: Waiting for close lock at 1732077005286Disabling compacts and flushes for region at 1732077005286Disabling writes for close at 1732077005287 (+1 ms)Writing region close event to WAL at 1732077005287Closed at 1732077005287 2024-11-20T04:30:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741924_1103 (size=93) 2024-11-20T04:30:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741924_1103 (size=93) 2024-11-20T04:30:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741924_1103 (size=93) 2024-11-20T04:30:05,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:30:05,292 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732077005222) 2024-11-20T04:30:05,307 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:42657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_675033121_22 at /127.0.0.1:38202 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:42657 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_675033121_22 at /127.0.0.1:56564 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6788 (was 6842) 2024-11-20T04:30:05,307 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-20T04:30:05,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=6788 2024-11-20T04:30:05,321 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-20T04:30:05,336 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:30:05,338 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T04:30:05,339 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T04:30:05,341 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32132307, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/hregion-32132307, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:05,355 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32132307/hregion-32132307.1732077005341, exclude list is [], retry=0 2024-11-20T04:30:05,358 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:05,358 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:05,358 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:05,360 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32132307/hregion-32132307.1732077005341 2024-11-20T04:30:05,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839)] 2024-11-20T04:30:05,361 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f36e56ab095c2285c3364936143359d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42657/hbase 2024-11-20T04:30:05,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741926_1105 (size=67) 2024-11-20T04:30:05,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741926_1105 (size=67) 2024-11-20T04:30:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741926_1105 (size=67) 2024-11-20T04:30:05,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:05,372 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,374 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName a 2024-11-20T04:30:05,374 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,374 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,374 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,375 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName b 2024-11-20T04:30:05,375 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,376 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,376 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,377 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName c 2024-11-20T04:30:05,377 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,377 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,379 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,379 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,380 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:05,381 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,383 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T04:30:05,384 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f36e56ab095c2285c3364936143359d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60949470, jitterRate=-0.09178212285041809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f36e56ab095c2285c3364936143359d9: Writing region info on filesystem at 1732077005370Initializing all the Stores at 1732077005370Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005370Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005372 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005372Cleaning up temporary data from old regions at 1732077005379 (+7 ms)Region opened successfully at 1732077005384 (+5 ms) 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f36e56ab095c2285c3364936143359d9, disabling compactions & flushes 2024-11-20T04:30:05,384 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. after waiting 0 ms 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,384 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f36e56ab095c2285c3364936143359d9: Waiting for close lock at 1732077005384Disabling compacts and flushes for region at 1732077005384Disabling writes for close at 1732077005384Writing region close event to WAL at 1732077005384Closed at 1732077005384 2024-11-20T04:30:05,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741925_1104 (size=93) 2024-11-20T04:30:05,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741925_1104 (size=93) 2024-11-20T04:30:05,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741925_1104 (size=93) 2024-11-20T04:30:05,389 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:30:05,389 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32132307:(num 1732077005341) 2024-11-20T04:30:05,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:05,391 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:05,405 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, exclude list is [], retry=0 2024-11-20T04:30:05,407 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:05,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:05,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:05,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 2024-11-20T04:30:05,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:30:05,410 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f36e56ab095c2285c3364936143359d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:05,410 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:05,410 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,410 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,412 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,412 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName a 2024-11-20T04:30:05,413 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,413 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,413 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,414 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName b 2024-11-20T04:30:05,414 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,414 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,415 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,415 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName c 2024-11-20T04:30:05,415 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,416 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,416 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,418 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,418 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,419 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:05,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f36e56ab095c2285c3364936143359d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69187990, jitterRate=0.030981391668319702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:05,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f36e56ab095c2285c3364936143359d9: Writing region info on filesystem at 1732077005410Initializing all the Stores at 1732077005411 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005411Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005411Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005411Cleaning up temporary data from old regions at 1732077005418 (+7 ms)Region opened successfully at 1732077005421 (+3 ms) 2024-11-20T04:30:05,429 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f36e56ab095c2285c3364936143359d9 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T04:30:05,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/7a7e353ea5484a308b994f7f0a7e6993 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732077005421/Put/seqid=0 2024-11-20T04:30:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741928_1107 (size=5958) 2024-11-20T04:30:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741928_1107 (size=5958) 2024-11-20T04:30:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741928_1107 (size=5958) 2024-11-20T04:30:05,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/7a7e353ea5484a308b994f7f0a7e6993 2024-11-20T04:30:05,460 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/7a7e353ea5484a308b994f7f0a7e6993 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/7a7e353ea5484a308b994f7f0a7e6993 2024-11-20T04:30:05,466 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/7a7e353ea5484a308b994f7f0a7e6993, entries=10, sequenceid=13, filesize=5.8 K 2024-11-20T04:30:05,467 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f36e56ab095c2285c3364936143359d9 in 38ms, sequenceid=13, compaction requested=false 2024-11-20T04:30:05,467 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f36e56ab095c2285c3364936143359d9: 2024-11-20T04:30:05,484 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f36e56ab095c2285c3364936143359d9, disabling compactions & flushes 2024-11-20T04:30:05,484 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,484 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,484 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. after waiting 0 ms 2024-11-20T04:30:05,484 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,485 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,485 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:05,485 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f36e56ab095c2285c3364936143359d9: Waiting for close lock at 1732077005484Disabling compacts and flushes for region at 1732077005484Disabling writes for close at 1732077005484Writing region close event to WAL at 1732077005485 (+1 ms)Closed at 1732077005485 2024-11-20T04:30:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741927_1106 (size=2805) 2024-11-20T04:30:05,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741927_1106 (size=2805) 2024-11-20T04:30:05,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741927_1106 (size=2805) 2024-11-20T04:30:05,504 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, size=2.7 K (2805bytes) 2024-11-20T04:30:05,504 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 2024-11-20T04:30:05,505 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 after 0ms 2024-11-20T04:30:05,507 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:05,508 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 took 3ms 2024-11-20T04:30:05,509 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 so closing down 2024-11-20T04:30:05,510 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:30:05,511 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732077005391.temp 2024-11-20T04:30:05,512 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp 2024-11-20T04:30:05,512 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:30:05,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741929_1108 (size=2312) 2024-11-20T04:30:05,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741929_1108 (size=2312) 2024-11-20T04:30:05,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741929_1108 (size=2312) 2024-11-20T04:30:05,522 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T04:30:05,524 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 2024-11-20T04:30:05,524 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-20T04:30:05,524 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, journal: Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, size=2.7 K (2805bytes) at 1732077005504Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 so closing down at 1732077005510 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp at 1732077005512 (+2 ms)3 split writer threads finished at 1732077005512Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732077005522 (+10 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000003-wal.1732077005391.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 at 1732077005524 (+2 ms)Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732077005524 2024-11-20T04:30:05,526 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005391 to hdfs://localhost:42657/hbase/oldWALs/wal.1732077005391 2024-11-20T04:30:05,527 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 2024-11-20T04:30:05,527 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:05,529 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:05,543 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, exclude list is [], retry=0 2024-11-20T04:30:05,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:05,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:05,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:05,548 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 2024-11-20T04:30:05,548 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40607:40607),(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-20T04:30:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f36e56ab095c2285c3364936143359d9, NAME => 'testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.', STARTKEY => '', ENDKEY => ''} 2024-11-20T04:30:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,551 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,552 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName a 2024-11-20T04:30:05,552 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,558 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/7a7e353ea5484a308b994f7f0a7e6993 2024-11-20T04:30:05,558 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,558 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,559 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName b 2024-11-20T04:30:05,559 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,559 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,560 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,561 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName c 2024-11-20T04:30:05,561 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:05,561 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:05,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,564 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,565 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 2024-11-20T04:30:05,567 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:05,569 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 2024-11-20T04:30:05,569 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f36e56ab095c2285c3364936143359d9 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-20T04:30:05,592 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/273779a9aa30412b823091cc19315dcc is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732077005467/Put/seqid=0 2024-11-20T04:30:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741931_1110 (size=5958) 2024-11-20T04:30:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741931_1110 (size=5958) 2024-11-20T04:30:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741931_1110 (size=5958) 2024-11-20T04:30:05,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/273779a9aa30412b823091cc19315dcc 2024-11-20T04:30:05,621 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/907a13c1193f444a89c6a5b2d142590b is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732077005474/Put/seqid=0 2024-11-20T04:30:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741932_1111 (size=5958) 2024-11-20T04:30:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741932_1111 (size=5958) 2024-11-20T04:30:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741932_1111 (size=5958) 2024-11-20T04:30:05,629 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/907a13c1193f444a89c6a5b2d142590b 2024-11-20T04:30:05,636 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/273779a9aa30412b823091cc19315dcc as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/b/273779a9aa30412b823091cc19315dcc 2024-11-20T04:30:05,642 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/b/273779a9aa30412b823091cc19315dcc, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T04:30:05,643 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/907a13c1193f444a89c6a5b2d142590b as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/c/907a13c1193f444a89c6a5b2d142590b 2024-11-20T04:30:05,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/c/907a13c1193f444a89c6a5b2d142590b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T04:30:05,650 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f36e56ab095c2285c3364936143359d9 in 81ms, sequenceid=35, compaction requested=false; wal=null 2024-11-20T04:30:05,651 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000035 2024-11-20T04:30:05,652 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,652 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,653 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:05,655 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:05,657 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-20T04:30:05,659 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f36e56ab095c2285c3364936143359d9; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63423551, jitterRate=-0.05491544306278229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:05,659 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f36e56ab095c2285c3364936143359d9: Writing region info on filesystem at 1732077005549Initializing all the Stores at 1732077005550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005550Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005551 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077005551Obtaining lock to block concurrent updates at 1732077005569 (+18 ms)Preparing flush snapshotting stores in f36e56ab095c2285c3364936143359d9 at 1732077005569Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732077005569Flushing stores of testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. at 1732077005569Flushing f36e56ab095c2285c3364936143359d9/b: creating writer at 1732077005569Flushing f36e56ab095c2285c3364936143359d9/b: appending metadata at 1732077005591 (+22 ms)Flushing f36e56ab095c2285c3364936143359d9/b: closing flushed file at 1732077005591Flushing f36e56ab095c2285c3364936143359d9/c: creating writer at 1732077005605 (+14 ms)Flushing f36e56ab095c2285c3364936143359d9/c: appending metadata at 1732077005621 (+16 ms)Flushing f36e56ab095c2285c3364936143359d9/c: closing flushed file at 1732077005621Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e1b9bd: reopening flushed file at 1732077005635 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7433ed0c: reopening flushed file at 1732077005642 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f36e56ab095c2285c3364936143359d9 in 81ms, sequenceid=35, compaction requested=false; wal=null at 1732077005650 (+8 ms)Cleaning up temporary data from old regions at 1732077005652 (+2 ms)Region opened successfully at 1732077005659 (+7 ms) 2024-11-20T04:30:05,737 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, size=0 (0bytes) 2024-11-20T04:30:05,737 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 might be still open, length is 0 2024-11-20T04:30:05,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 2024-11-20T04:30:05,737 WARN [IPC Server handler 1 on default port 42657 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-20T04:30:05,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 after 1ms 2024-11-20T04:30:08,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:38260 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45985:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38260 dst: /127.0.0.1:45985 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45985 remote=/127.0.0.1:38260]. Total timeout mills is 60000, 57304 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:08,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:56630 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46091:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56630 dst: /127.0.0.1:46091 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:08,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1948416810_22 at /127.0.0.1:42500 [Receiving block BP-2101640850-172.17.0.2-1732076966903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42500 dst: /127.0.0.1:43305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741930_1112 (size=2304) 2024-11-20T04:30:08,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741930_1112 (size=2304) 2024-11-20T04:30:08,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741930_1112 (size=2304) 2024-11-20T04:30:09,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 after 4001ms 2024-11-20T04:30:09,741 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:09,742 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 took 4006ms 2024-11-20T04:30:09,744 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529; continuing. 2024-11-20T04:30:09,744 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 so closing down 2024-11-20T04:30:09,744 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T04:30:09,745 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732077005529.temp 2024-11-20T04:30:09,747 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp 2024-11-20T04:30:09,747 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T04:30:09,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741933_1113 (size=2312) 2024-11-20T04:30:09,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741933_1113 (size=2312) 2024-11-20T04:30:09,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741933_1113 (size=2312) 2024-11-20T04:30:09,759 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T04:30:09,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 2024-11-20T04:30:09,761 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T04:30:09,761 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, journal: Splitting hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, size=0 (0bytes) at 1732077005737Finishing writing output for hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 so closing down at 1732077009744 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp at 1732077009747 (+3 ms)3 split writer threads finished at 1732077009747Closed recovered edits writer path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732077009759 (+12 ms)Rename recovered edits hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000037-wal.1732077005529.temp to hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 at 1732077009761 (+2 ms)Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529, size=0, length=0, corrupted=false, cancelled=false at 1732077009761 2024-11-20T04:30:09,763 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 to hdfs://localhost:42657/hbase/oldWALs/wal.1732077005529 2024-11-20T04:30:09,764 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 2024-11-20T04:30:09,764 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T04:30:09,766 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42657/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335, archiveDir=hdfs://localhost:42657/hbase/oldWALs, maxLogs=32 2024-11-20T04:30:09,779 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077009766, exclude list is [], retry=0 2024-11-20T04:30:09,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43305,DS-2bf7ec94-06e8-4123-a913-acf570aee653,DISK] 2024-11-20T04:30:09,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46091,DS-f69e25c5-f43a-4837-a63a-a76ac09a4f9c,DISK] 2024-11-20T04:30:09,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45985,DS-ffd0d97e-8f25-4bcf-814b-58c4e5b3434b,DISK] 2024-11-20T04:30:09,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077009766 2024-11-20T04:30:09,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41839:41839),(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:40607:40607)] 2024-11-20T04:30:09,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T04:30:09,786 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,787 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName a 2024-11-20T04:30:09,787 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:09,818 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/7a7e353ea5484a308b994f7f0a7e6993 2024-11-20T04:30:09,818 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:09,818 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,819 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName b 2024-11-20T04:30:09,819 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:09,825 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/b/273779a9aa30412b823091cc19315dcc 2024-11-20T04:30:09,826 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:09,826 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,826 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f36e56ab095c2285c3364936143359d9 columnFamilyName c 2024-11-20T04:30:09,827 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T04:30:09,831 DEBUG [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/c/907a13c1193f444a89c6a5b2d142590b 2024-11-20T04:30:09,832 INFO [StoreOpener-f36e56ab095c2285c3364936143359d9-1 {}] regionserver.HStore(327): Store=f36e56ab095c2285c3364936143359d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T04:30:09,832 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,833 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,834 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,835 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 2024-11-20T04:30:09,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T04:30:09,842 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 2024-11-20T04:30:09,842 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f36e56ab095c2285c3364936143359d9 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T04:30:09,858 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/21d742629c974119b9d79bd98b776ac3 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732077005667/Put/seqid=0 2024-11-20T04:30:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741935_1115 (size=5958) 2024-11-20T04:30:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741935_1115 (size=5958) 2024-11-20T04:30:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741935_1115 (size=5958) 2024-11-20T04:30:09,865 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/21d742629c974119b9d79bd98b776ac3 2024-11-20T04:30:09,885 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/43add971fca34522b4f5d87ffd747d31 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732077005675/Put/seqid=0 2024-11-20T04:30:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741936_1116 (size=5958) 2024-11-20T04:30:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741936_1116 (size=5958) 2024-11-20T04:30:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741936_1116 (size=5958) 2024-11-20T04:30:09,892 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/43add971fca34522b4f5d87ffd747d31 2024-11-20T04:30:09,912 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/3711682a42664b51a2abaa5e3ff521c3 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732077005683/Put/seqid=0 2024-11-20T04:30:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741937_1117 (size=5958) 2024-11-20T04:30:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741937_1117 (size=5958) 2024-11-20T04:30:09,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741937_1117 (size=5958) 2024-11-20T04:30:09,919 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/3711682a42664b51a2abaa5e3ff521c3 2024-11-20T04:30:09,925 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/a/21d742629c974119b9d79bd98b776ac3 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/21d742629c974119b9d79bd98b776ac3 2024-11-20T04:30:09,930 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/a/21d742629c974119b9d79bd98b776ac3, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T04:30:09,930 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/b/43add971fca34522b4f5d87ffd747d31 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/b/43add971fca34522b4f5d87ffd747d31 2024-11-20T04:30:09,936 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/b/43add971fca34522b4f5d87ffd747d31, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T04:30:09,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/.tmp/c/3711682a42664b51a2abaa5e3ff521c3 as hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/c/3711682a42664b51a2abaa5e3ff521c3 2024-11-20T04:30:09,941 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/c/3711682a42664b51a2abaa5e3ff521c3, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T04:30:09,942 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f36e56ab095c2285c3364936143359d9 in 99ms, sequenceid=66, compaction requested=false; wal=null 2024-11-20T04:30:09,942 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/0000000000000000066 2024-11-20T04:30:09,943 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,943 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,944 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T04:30:09,945 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f36e56ab095c2285c3364936143359d9 2024-11-20T04:30:09,947 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/hbase/data/default/testReplayEditsWrittenViaHRegion/f36e56ab095c2285c3364936143359d9/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-20T04:30:09,948 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f36e56ab095c2285c3364936143359d9; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68249895, jitterRate=0.017002686858177185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T04:30:09,948 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f36e56ab095c2285c3364936143359d9: Writing region info on filesystem at 1732077009785Initializing all the Stores at 1732077009785Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077009785Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077009786 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732077009786Obtaining lock to block concurrent updates at 1732077009842 (+56 ms)Preparing flush snapshotting stores in f36e56ab095c2285c3364936143359d9 at 1732077009842Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732077009842Flushing stores of testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. at 1732077009842Flushing f36e56ab095c2285c3364936143359d9/a: creating writer at 1732077009842Flushing f36e56ab095c2285c3364936143359d9/a: appending metadata at 1732077009857 (+15 ms)Flushing f36e56ab095c2285c3364936143359d9/a: closing flushed file at 1732077009857Flushing f36e56ab095c2285c3364936143359d9/b: creating writer at 1732077009870 (+13 ms)Flushing f36e56ab095c2285c3364936143359d9/b: appending metadata at 1732077009884 (+14 ms)Flushing f36e56ab095c2285c3364936143359d9/b: closing flushed file at 1732077009884Flushing f36e56ab095c2285c3364936143359d9/c: creating writer at 1732077009897 (+13 ms)Flushing f36e56ab095c2285c3364936143359d9/c: appending metadata at 1732077009911 (+14 ms)Flushing f36e56ab095c2285c3364936143359d9/c: closing flushed file at 1732077009911Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6df84f54: reopening flushed file at 1732077009924 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@515f7407: reopening flushed file at 1732077009930 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbbfc83: reopening flushed file at 1732077009936 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f36e56ab095c2285c3364936143359d9 in 99ms, sequenceid=66, compaction requested=false; wal=null at 1732077009942 (+6 ms)Cleaning up temporary data from old regions at 1732077009943 (+1 ms)Region opened successfully at 1732077009948 (+5 ms) 2024-11-20T04:30:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f36e56ab095c2285c3364936143359d9, disabling compactions & flushes 2024-11-20T04:30:09,963 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. after waiting 0 ms 2024-11-20T04:30:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:09,966 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732077005337.f36e56ab095c2285c3364936143359d9. 2024-11-20T04:30:09,966 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f36e56ab095c2285c3364936143359d9: Waiting for close lock at 1732077009963Disabling compacts and flushes for region at 1732077009963Disabling writes for close at 1732077009963Writing region close event to WAL at 1732077009966 (+3 ms)Closed at 1732077009966 2024-11-20T04:30:09,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741934_1114 (size=93) 2024-11-20T04:30:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741934_1114 (size=93) 2024-11-20T04:30:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741934_1114 (size=93) 2024-11-20T04:30:09,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T04:30:09,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732077009766) 2024-11-20T04:30:09,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=444 (was 438) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:42657 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_574654101_22 at /127.0.0.1:38318 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1640860714) connection to localhost/127.0.0.1:42657 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_574654101_22 at /127.0.0.1:42534 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_574654101_22 at /127.0.0.1:56672 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=212 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=6787 (was 6788) 2024-11-20T04:30:09,993 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-20T04:30:09,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T04:30:09,994 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T04:30:09,994 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T04:30:09,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:09,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:09,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T04:30:09,995 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T04:30:09,995 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223486682, stopped=false 2024-11-20T04:30:09,995 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2a32e16c274,37809,1732076970521 2024-11-20T04:30:09,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T04:30:09,999 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T04:30:09,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:30:09,999 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:30:09,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T04:30:09,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:30:09,999 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T04:30:10,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:30:10,000 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T04:30:10,000 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T04:30:10,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:10,000 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2a32e16c274,38897,1732076971421' ***** 2024-11-20T04:30:10,000 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T04:30:10,000 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2a32e16c274,39781,1732076971499' ***** 2024-11-20T04:30:10,001 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(959): stopping server c2a32e16c274,39781,1732076971499 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(3091): Received CLOSE for 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c2a32e16c274:39781. 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(959): stopping server c2a32e16c274,38897,1732076971421 2024-11-20T04:30:10,001 DEBUG [RS:2;c2a32e16c274:39781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T04:30:10,001 DEBUG [RS:2;c2a32e16c274:39781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:10,001 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(976): stopping server c2a32e16c274,39781,1732076971499; all regions closed. 2024-11-20T04:30:10,001 INFO [RS:1;c2a32e16c274:38897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c2a32e16c274:38897. 2024-11-20T04:30:10,001 DEBUG [RS:1;c2a32e16c274:38897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T04:30:10,002 DEBUG [RS:1;c2a32e16c274:38897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:10,001 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46816b69831c7c3d004fa2452f463d5c, disabling compactions & flushes 2024-11-20T04:30:10,002 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:30:10,002 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T04:30:10,002 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T04:30:10,002 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:30:10,002 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T04:30:10,002 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. after waiting 0 ms 2024-11-20T04:30:10,002 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:30:10,002 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T04:30:10,002 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T04:30:10,002 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T04:30:10,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:30:10,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T04:30:10,005 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T04:30:10,005 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1325): Online Regions={46816b69831c7c3d004fa2452f463d5c=testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T04:30:10,005 DEBUG [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46816b69831c7c3d004fa2452f463d5c 2024-11-20T04:30:10,005 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T04:30:10,005 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T04:30:10,005 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T04:30:10,005 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T04:30:10,005 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T04:30:10,006 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-20T04:30:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741834_1010 (size=93) 2024-11-20T04:30:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741834_1010 (size=93) 2024-11-20T04:30:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741834_1010 (size=93) 2024-11-20T04:30:10,011 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/46816b69831c7c3d004fa2452f463d5c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-20T04:30:10,012 DEBUG [RS:2;c2a32e16c274:39781 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs 2024-11-20T04:30:10,012 INFO [RS:2;c2a32e16c274:39781 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2a32e16c274%2C39781%2C1732076971499:(num 1732076973120) 2024-11-20T04:30:10,012 DEBUG [RS:2;c2a32e16c274:39781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:10,012 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:30:10,012 INFO [RS:2;c2a32e16c274:39781 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T04:30:10,012 INFO [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:30:10,012 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46816b69831c7c3d004fa2452f463d5c: Waiting for close lock at 1732077010001Running coprocessor pre-close hooks at 1732077010001Disabling compacts and flushes for region at 1732077010001Disabling writes for close at 1732077010002 (+1 ms)Writing region close event to WAL at 1732077010006 (+4 ms)Running coprocessor post-close hooks at 1732077010012 (+6 ms)Closed at 1732077010012 2024-11-20T04:30:10,012 INFO [RS:2;c2a32e16c274:39781 {}] hbase.ChoreService(370): Chore service for: regionserver/c2a32e16c274:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T04:30:10,013 DEBUG [RS_CLOSE_REGION-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c. 2024-11-20T04:30:10,013 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T04:30:10,013 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T04:30:10,013 INFO [regionserver/c2a32e16c274:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T04:30:10,013 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T04:30:10,013 INFO [RS:2;c2a32e16c274:39781 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T04:30:10,013 INFO [RS:2;c2a32e16c274:39781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39781 2024-11-20T04:30:10,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T04:30:10,017 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2a32e16c274,39781,1732076971499 2024-11-20T04:30:10,017 INFO [RS:2;c2a32e16c274:39781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T04:30:10,017 INFO [regionserver/c2a32e16c274:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:30:10,017 INFO [regionserver/c2a32e16c274:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:30:10,018 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2a32e16c274,39781,1732076971499] 2024-11-20T04:30:10,019 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2a32e16c274,39781,1732076971499 already deleted, retry=false 2024-11-20T04:30:10,019 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2a32e16c274,39781,1732076971499 expired; onlineServers=1 2024-11-20T04:30:10,032 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/info/58830ad3bea44e779c08f1409995f52b is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732076986329.46816b69831c7c3d004fa2452f463d5c./info:regioninfo/1732076989405/Put/seqid=0 2024-11-20T04:30:10,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741938_1118 (size=8243) 2024-11-20T04:30:10,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741938_1118 (size=8243) 2024-11-20T04:30:10,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741938_1118 (size=8243) 2024-11-20T04:30:10,042 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/info/58830ad3bea44e779c08f1409995f52b 2024-11-20T04:30:10,063 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/ns/3ea85ad3344a4315a9cb0dcf9e69236f is 43, key is default/ns:d/1732076973766/Put/seqid=0 2024-11-20T04:30:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741939_1119 (size=5153) 2024-11-20T04:30:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741939_1119 (size=5153) 2024-11-20T04:30:10,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741939_1119 (size=5153) 2024-11-20T04:30:10,071 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/ns/3ea85ad3344a4315a9cb0dcf9e69236f 2024-11-20T04:30:10,092 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/table/134f7eae58444ee4a376f6ff905c9d0b is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732076986744/Put/seqid=0 2024-11-20T04:30:10,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741940_1120 (size=5431) 2024-11-20T04:30:10,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741940_1120 (size=5431) 2024-11-20T04:30:10,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741940_1120 (size=5431) 2024-11-20T04:30:10,098 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/table/134f7eae58444ee4a376f6ff905c9d0b 2024-11-20T04:30:10,105 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/info/58830ad3bea44e779c08f1409995f52b as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/info/58830ad3bea44e779c08f1409995f52b 2024-11-20T04:30:10,112 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/info/58830ad3bea44e779c08f1409995f52b, entries=18, sequenceid=21, filesize=8.0 K 2024-11-20T04:30:10,113 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/ns/3ea85ad3344a4315a9cb0dcf9e69236f as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/ns/3ea85ad3344a4315a9cb0dcf9e69236f 2024-11-20T04:30:10,118 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,118 INFO [RS:2;c2a32e16c274:39781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T04:30:10,118 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39781-0x10133a3095f0003, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,119 INFO [RS:2;c2a32e16c274:39781 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2a32e16c274,39781,1732076971499; zookeeper connection closed. 2024-11-20T04:30:10,119 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@797c00bd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@797c00bd 2024-11-20T04:30:10,121 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/ns/3ea85ad3344a4315a9cb0dcf9e69236f, entries=2, sequenceid=21, filesize=5.0 K 2024-11-20T04:30:10,122 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/.tmp/table/134f7eae58444ee4a376f6ff905c9d0b as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/table/134f7eae58444ee4a376f6ff905c9d0b 2024-11-20T04:30:10,128 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/table/134f7eae58444ee4a376f6ff905c9d0b, entries=2, sequenceid=21, filesize=5.3 K 2024-11-20T04:30:10,130 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=21, compaction requested=false 2024-11-20T04:30:10,138 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-20T04:30:10,139 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T04:30:10,140 INFO [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T04:30:10,140 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732077010005Running coprocessor pre-close hooks at 1732077010005Disabling compacts and flushes for region at 1732077010005Disabling writes for close at 1732077010005Obtaining lock to block concurrent updates at 1732077010006 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732077010006Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732077010006Flushing stores of hbase:meta,,1.1588230740 at 1732077010008 (+2 ms)Flushing 1588230740/info: creating writer at 1732077010008Flushing 1588230740/info: appending metadata at 1732077010031 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732077010031Flushing 1588230740/ns: creating writer at 1732077010048 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732077010063 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732077010063Flushing 1588230740/table: creating writer at 1732077010077 (+14 ms)Flushing 1588230740/table: appending metadata at 1732077010091 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732077010091Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@519748b: reopening flushed file at 1732077010104 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b3dd1fc: reopening flushed file at 1732077010112 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f3a178e: reopening flushed file at 1732077010121 (+9 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=21, compaction requested=false at 1732077010130 (+9 ms)Writing region close event to WAL at 1732077010132 (+2 ms)Running coprocessor post-close hooks at 1732077010139 (+7 ms)Closed at 1732077010140 (+1 ms) 2024-11-20T04:30:10,140 DEBUG [RS_CLOSE_META-regionserver/c2a32e16c274:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T04:30:10,205 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(976): stopping server c2a32e16c274,38897,1732076971421; all regions closed. 2024-11-20T04:30:10,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741836_1012 (size=3783) 2024-11-20T04:30:10,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741836_1012 (size=3783) 2024-11-20T04:30:10,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741836_1012 (size=3783) 2024-11-20T04:30:10,211 DEBUG [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs 2024-11-20T04:30:10,211 INFO [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2a32e16c274%2C38897%2C1732076971421.meta:.meta(num 1732076973591) 2024-11-20T04:30:10,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741835_1011 (size=2652) 2024-11-20T04:30:10,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741835_1011 (size=2652) 2024-11-20T04:30:10,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741835_1011 (size=2652) 2024-11-20T04:30:10,215 DEBUG [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/oldWALs 2024-11-20T04:30:10,215 INFO [RS:1;c2a32e16c274:38897 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2a32e16c274%2C38897%2C1732076971421:(num 1732076973120) 2024-11-20T04:30:10,215 DEBUG [RS:1;c2a32e16c274:38897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T04:30:10,215 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T04:30:10,216 INFO [RS:1;c2a32e16c274:38897 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T04:30:10,216 INFO [RS:1;c2a32e16c274:38897 {}] hbase.ChoreService(370): Chore service for: regionserver/c2a32e16c274:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T04:30:10,216 INFO [RS:1;c2a32e16c274:38897 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T04:30:10,216 INFO [regionserver/c2a32e16c274:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T04:30:10,216 INFO [RS:1;c2a32e16c274:38897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38897 2024-11-20T04:30:10,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T04:30:10,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2a32e16c274,38897,1732076971421 2024-11-20T04:30:10,218 INFO [RS:1;c2a32e16c274:38897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T04:30:10,223 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2a32e16c274,38897,1732076971421] 2024-11-20T04:30:10,224 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2a32e16c274,38897,1732076971421 already deleted, retry=false 2024-11-20T04:30:10,224 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2a32e16c274,38897,1732076971421 expired; onlineServers=0 2024-11-20T04:30:10,224 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2a32e16c274,37809,1732076970521' ***** 2024-11-20T04:30:10,224 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T04:30:10,224 INFO [M:0;c2a32e16c274:37809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T04:30:10,224 INFO [M:0;c2a32e16c274:37809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T04:30:10,224 DEBUG [M:0;c2a32e16c274:37809 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T04:30:10,225 DEBUG [M:0;c2a32e16c274:37809 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T04:30:10,225 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T04:30:10,225 DEBUG [master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.small.0-1732076972767 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.small.0-1732076972767,5,FailOnTimeoutGroup] 2024-11-20T04:30:10,225 DEBUG [master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.large.0-1732076972766 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2a32e16c274:0:becomeActiveMaster-HFileCleaner.large.0-1732076972766,5,FailOnTimeoutGroup] 2024-11-20T04:30:10,225 INFO [M:0;c2a32e16c274:37809 {}] hbase.ChoreService(370): Chore service for: master/c2a32e16c274:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T04:30:10,225 INFO [M:0;c2a32e16c274:37809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T04:30:10,225 DEBUG [M:0;c2a32e16c274:37809 {}] master.HMaster(1795): Stopping service threads 2024-11-20T04:30:10,225 INFO [M:0;c2a32e16c274:37809 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T04:30:10,225 INFO [M:0;c2a32e16c274:37809 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T04:30:10,226 INFO [M:0;c2a32e16c274:37809 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T04:30:10,226 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T04:30:10,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T04:30:10,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T04:30:10,227 DEBUG [M:0;c2a32e16c274:37809 {}] zookeeper.ZKUtil(347): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T04:30:10,227 WARN [M:0;c2a32e16c274:37809 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T04:30:10,227 INFO [M:0;c2a32e16c274:37809 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/.lastflushedseqids 2024-11-20T04:30:10,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741941_1121 (size=138) 2024-11-20T04:30:10,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741941_1121 (size=138) 2024-11-20T04:30:10,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741941_1121 (size=138) 2024-11-20T04:30:10,241 INFO [M:0;c2a32e16c274:37809 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T04:30:10,241 INFO [M:0;c2a32e16c274:37809 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T04:30:10,241 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T04:30:10,241 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:30:10,241 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:30:10,241 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T04:30:10,241 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:30:10,241 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-20T04:30:10,258 DEBUG [M:0;c2a32e16c274:37809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c261bf562ae4ce6808dabe6b7984e73 is 82, key is hbase:meta,,1/info:regioninfo/1732076973683/Put/seqid=0 2024-11-20T04:30:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741942_1122 (size=5672) 2024-11-20T04:30:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741942_1122 (size=5672) 2024-11-20T04:30:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741942_1122 (size=5672) 2024-11-20T04:30:10,265 INFO [M:0;c2a32e16c274:37809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c261bf562ae4ce6808dabe6b7984e73 2024-11-20T04:30:10,284 DEBUG [M:0;c2a32e16c274:37809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2df75f2907145cbbd621a49157868c9 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732076986750/Put/seqid=0 2024-11-20T04:30:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741943_1123 (size=7755) 2024-11-20T04:30:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741943_1123 (size=7755) 2024-11-20T04:30:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741943_1123 (size=7755) 2024-11-20T04:30:10,292 INFO [M:0;c2a32e16c274:37809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2df75f2907145cbbd621a49157868c9 2024-11-20T04:30:10,297 INFO [M:0;c2a32e16c274:37809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2df75f2907145cbbd621a49157868c9 2024-11-20T04:30:10,318 DEBUG [M:0;c2a32e16c274:37809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/adf3b67c580d4b94b4efbb3c124e2cff is 69, key is c2a32e16c274,38897,1732076971421/rs:state/1732076972835/Put/seqid=0 2024-11-20T04:30:10,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x10133a3095f0002, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,323 INFO [RS:1;c2a32e16c274:38897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T04:30:10,323 INFO [RS:1;c2a32e16c274:38897 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2a32e16c274,38897,1732076971421; zookeeper connection closed. 2024-11-20T04:30:10,323 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fa294cb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fa294cb 2024-11-20T04:30:10,324 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T04:30:10,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741944_1124 (size=5440) 2024-11-20T04:30:10,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741944_1124 (size=5440) 2024-11-20T04:30:10,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741944_1124 (size=5440) 2024-11-20T04:30:10,326 INFO [M:0;c2a32e16c274:37809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/adf3b67c580d4b94b4efbb3c124e2cff 2024-11-20T04:30:10,331 INFO [M:0;c2a32e16c274:37809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for adf3b67c580d4b94b4efbb3c124e2cff 2024-11-20T04:30:10,332 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c261bf562ae4ce6808dabe6b7984e73 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4c261bf562ae4ce6808dabe6b7984e73 2024-11-20T04:30:10,337 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4c261bf562ae4ce6808dabe6b7984e73, entries=8, sequenceid=168, filesize=5.5 K 2024-11-20T04:30:10,339 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2df75f2907145cbbd621a49157868c9 as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2df75f2907145cbbd621a49157868c9 2024-11-20T04:30:10,344 INFO [M:0;c2a32e16c274:37809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2df75f2907145cbbd621a49157868c9 2024-11-20T04:30:10,344 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2df75f2907145cbbd621a49157868c9, entries=17, sequenceid=168, filesize=7.6 K 2024-11-20T04:30:10,345 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/adf3b67c580d4b94b4efbb3c124e2cff as hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/adf3b67c580d4b94b4efbb3c124e2cff 2024-11-20T04:30:10,351 INFO [M:0;c2a32e16c274:37809 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for adf3b67c580d4b94b4efbb3c124e2cff 2024-11-20T04:30:10,352 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42657/user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/adf3b67c580d4b94b4efbb3c124e2cff, entries=3, sequenceid=168, filesize=5.3 K 2024-11-20T04:30:10,354 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=168, compaction requested=false 2024-11-20T04:30:10,356 INFO [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T04:30:10,357 DEBUG [M:0;c2a32e16c274:37809 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732077010241Disabling compacts and flushes for region at 1732077010241Disabling writes for close at 1732077010241Obtaining lock to block concurrent updates at 1732077010241Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732077010241Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732077010242 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732077010242Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732077010243 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732077010257 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732077010257Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732077010269 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732077010284 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732077010284Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732077010298 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732077010318 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732077010318Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4adbd415: reopening flushed file at 1732077010331 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2790fe6f: reopening flushed file at 1732077010337 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c712b59: reopening flushed file at 1732077010345 (+8 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=168, compaction requested=false at 1732077010354 (+9 ms)Writing region close event to WAL at 1732077010356 (+2 ms)Closed at 1732077010356 2024-11-20T04:30:10,361 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3e8468e1-e900-7350-9035-f73c91a9103e/MasterData/WALs/c2a32e16c274,37809,1732076970521/c2a32e16c274%2C37809%2C1732076970521.1732076972031 not finished, retry = 0 2024-11-20T04:30:10,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46091 is added to blk_1073741830_1006 (size=71337) 2024-11-20T04:30:10,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45985 is added to blk_1073741830_1006 (size=71337) 2024-11-20T04:30:10,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43305 is added to blk_1073741830_1006 (size=71337) 2024-11-20T04:30:10,462 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T04:30:10,462 INFO [M:0;c2a32e16c274:37809 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T04:30:10,462 INFO [M:0;c2a32e16c274:37809 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37809 2024-11-20T04:30:10,463 INFO [M:0;c2a32e16c274:37809 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T04:30:10,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,565 INFO [M:0;c2a32e16c274:37809 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T04:30:10,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37809-0x10133a3095f0000, quorum=127.0.0.1:60520, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T04:30:10,571 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076990112 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076990112 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732076989799/wal.1732076990112 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T04:30:10,572 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732076981826/wal.1732076982014 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:10,572 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732077005335/wal.1732077005529 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:10,575 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076999600 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076999600 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732076990266/wal.1732076999600 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T04:30:10,575 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732076974988/wal.1732076975076 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:10,578 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732077000384 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732077000384 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732076999791/wal.1732077000384 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T04:30:10,578 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732077000531/wal.1732077000992 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T04:30:10,580 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal.1732076974830 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal.1732076974830 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732076974618/wal.1732076974830 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T04:30:10,583 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal.1732076974332 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal.1732076974332 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732076974081/wal.1732076974332 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1948416810_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T04:30:10,588 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f8545c0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:30:10,590 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@664ea373{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T04:30:10,590 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T04:30:10,591 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T04:30:10,591 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7701b046{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,STOPPED} 2024-11-20T04:30:10,594 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T04:30:10,594 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T04:30:10,594 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2101640850-172.17.0.2-1732076966903 (Datanode Uuid 6d34e9dd-5552-4279-8dab-2c7dce9ec7bf) service to localhost/127.0.0.1:42657 2024-11-20T04:30:10,594 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T04:30:10,595 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data5/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,595 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data6/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,596 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T04:30:10,598 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26760fc8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:30:10,598 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26b9ecc1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T04:30:10,598 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T04:30:10,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T04:30:10,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f53d973{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,STOPPED} 2024-11-20T04:30:10,600 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T04:30:10,600 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T04:30:10,600 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2101640850-172.17.0.2-1732076966903 (Datanode Uuid 50d4a516-a147-488b-84ab-55496d172715) service to localhost/127.0.0.1:42657 2024-11-20T04:30:10,600 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T04:30:10,601 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data3/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,601 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data4/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,601 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T04:30:10,603 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4da465b6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T04:30:10,603 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c26fca4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T04:30:10,603 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T04:30:10,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T04:30:10,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55114173{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,STOPPED} 2024-11-20T04:30:10,605 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T04:30:10,605 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T04:30:10,605 WARN [BP-2101640850-172.17.0.2-1732076966903 heartbeating to localhost/127.0.0.1:42657 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2101640850-172.17.0.2-1732076966903 (Datanode Uuid 61f47230-ad4a-4c9f-a628-a45306f25a26) service to localhost/127.0.0.1:42657 2024-11-20T04:30:10,605 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T04:30:10,606 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data1/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,606 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/cluster_24e4eb17-3690-59c4-0a80-e0fda07a31d6/data/data2/current/BP-2101640850-172.17.0.2-1732076966903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T04:30:10,606 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T04:30:10,613 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T04:30:10,613 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T04:30:10,613 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T04:30:10,614 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T04:30:10,614 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/67b6d991-f2b2-eb5a-477d-dac104e34cfc/hadoop.log.dir/,STOPPED} 2024-11-20T04:30:10,624 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T04:30:10,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down