2024-11-20 10:30:48,454 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-20 10:30:48,465 main DEBUG Took 0.009266 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 10:30:48,466 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 10:30:48,466 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 10:30:48,467 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 10:30:48,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,475 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 10:30:48,487 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,488 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,489 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,490 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,491 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,492 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,494 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,495 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,496 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 10:30:48,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,498 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 10:30:48,499 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 10:30:48,500 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 10:30:48,502 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 10:30:48,502 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 10:30:48,504 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 10:30:48,504 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 10:30:48,512 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 10:30:48,514 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 10:30:48,516 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 10:30:48,516 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 10:30:48,516 main DEBUG createAppenders(={Console}) 2024-11-20 10:30:48,517 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-20 10:30:48,517 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-20 10:30:48,517 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-20 10:30:48,518 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 10:30:48,518 main DEBUG OutputStream closed 2024-11-20 10:30:48,518 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 10:30:48,519 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 10:30:48,519 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-20 10:30:48,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 10:30:48,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 10:30:48,591 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 10:30:48,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 10:30:48,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 10:30:48,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 10:30:48,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 10:30:48,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 10:30:48,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 10:30:48,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 10:30:48,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 10:30:48,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 10:30:48,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 10:30:48,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 10:30:48,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 10:30:48,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 10:30:48,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 10:30:48,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 10:30:48,599 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 10:30:48,599 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-20 10:30:48,599 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 10:30:48,600 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-20T10:30:48,827 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d 2024-11-20 10:30:48,829 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 10:30:48,829 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T10:30:48,838 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-20T10:30:48,864 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T10:30:48,918 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T10:30:48,918 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T10:30:48,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T10:30:48,956 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3, deleteOnExit=true 2024-11-20T10:30:48,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T10:30:48,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/test.cache.data in system properties and HBase conf 2024-11-20T10:30:48,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T10:30:48,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir in system properties and HBase conf 2024-11-20T10:30:48,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T10:30:48,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T10:30:48,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T10:30:49,041 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T10:30:49,135 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T10:30:49,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T10:30:49,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T10:30:49,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T10:30:49,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T10:30:49,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T10:30:49,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T10:30:49,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T10:30:49,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T10:30:49,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T10:30:49,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/nfs.dump.dir in system properties and HBase conf 2024-11-20T10:30:49,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/java.io.tmpdir in system properties and HBase conf 2024-11-20T10:30:49,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T10:30:49,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T10:30:49,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T10:30:50,421 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T10:30:50,487 INFO [Time-limited test {}] log.Log(170): Logging initialized @2710ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T10:30:50,552 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:30:50,612 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:30:50,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:30:50,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:30:50,638 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T10:30:50,649 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:30:50,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:30:50,653 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:30:50,825 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/java.io.tmpdir/jetty-localhost-43991-hadoop-hdfs-3_4_1-tests_jar-_-any-2798726169769214461/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T10:30:50,832 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:43991} 2024-11-20T10:30:50,832 INFO [Time-limited test {}] server.Server(415): Started @3055ms 2024-11-20T10:30:51,396 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:30:51,403 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:30:51,404 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:30:51,404 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:30:51,405 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T10:30:51,405 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:30:51,406 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:30:51,505 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/java.io.tmpdir/jetty-localhost-43561-hadoop-hdfs-3_4_1-tests_jar-_-any-8112615505599071627/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:30:51,505 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:43561} 2024-11-20T10:30:51,506 INFO [Time-limited test {}] server.Server(415): Started @3729ms 2024-11-20T10:30:51,551 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T10:30:51,662 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:30:51,669 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:30:51,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:30:51,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:30:51,672 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T10:30:51,674 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:30:51,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:30:51,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/java.io.tmpdir/jetty-localhost-40865-hadoop-hdfs-3_4_1-tests_jar-_-any-5324491095551630235/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:30:51,786 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:40865} 2024-11-20T10:30:51,787 INFO [Time-limited test {}] server.Server(415): Started @4010ms 2024-11-20T10:30:51,790 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T10:30:51,830 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T10:30:51,835 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T10:30:51,837 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T10:30:51,837 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T10:30:51,837 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T10:30:51,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,AVAILABLE} 2024-11-20T10:30:51,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T10:30:51,938 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/java.io.tmpdir/jetty-localhost-43161-hadoop-hdfs-3_4_1-tests_jar-_-any-15745261140236859753/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:30:51,939 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:43161} 2024-11-20T10:30:51,939 INFO [Time-limited test {}] server.Server(415): Started @4162ms 2024-11-20T10:30:51,942 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T10:30:53,346 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data3/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,346 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data2/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,346 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data1/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,346 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data4/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,385 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T10:30:53,385 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T10:30:53,388 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data5/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,389 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data6/current/BP-1354523497-172.17.0.2-1732098649661/current, will proceed with Du for space computation calculation, 2024-11-20T10:30:53,413 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T10:30:53,430 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef4e373579d24a74 with lease ID 0x2b2881703bc15c: Processing first storage report for DS-598a40d1-0b51-4df7-942d-94be907a94aa from datanode DatanodeRegistration(127.0.0.1:39533, datanodeUuid=056dc2d8-52c9-4108-b42e-d3d4b78ab6f2, infoPort=34305, infoSecurePort=0, ipcPort=43961, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef4e373579d24a74 with lease ID 0x2b2881703bc15c: from storage DS-598a40d1-0b51-4df7-942d-94be907a94aa node DatanodeRegistration(127.0.0.1:39533, datanodeUuid=056dc2d8-52c9-4108-b42e-d3d4b78ab6f2, infoPort=34305, infoSecurePort=0, ipcPort=43961, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7abb297e25a0b74 with lease ID 0x2b2881703bc15d: Processing first storage report for DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc from datanode DatanodeRegistration(127.0.0.1:38347, datanodeUuid=fb294b86-2cb6-45be-b146-b1a7befb00f0, infoPort=41233, infoSecurePort=0, ipcPort=41885, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7abb297e25a0b74 with lease ID 0x2b2881703bc15d: from storage DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc node DatanodeRegistration(127.0.0.1:38347, datanodeUuid=fb294b86-2cb6-45be-b146-b1a7befb00f0, infoPort=41233, infoSecurePort=0, ipcPort=41885, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b5cb8657a859c77 with lease ID 0x2b2881703bc15e: Processing first storage report for DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347 from datanode DatanodeRegistration(127.0.0.1:46093, datanodeUuid=d1970de4-eae2-4112-a0d1-f0d815042ff4, infoPort=34509, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b5cb8657a859c77 with lease ID 0x2b2881703bc15e: from storage DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347 node DatanodeRegistration(127.0.0.1:46093, datanodeUuid=d1970de4-eae2-4112-a0d1-f0d815042ff4, infoPort=34509, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xef4e373579d24a74 with lease ID 0x2b2881703bc15c: Processing first storage report for DS-49fdbf28-adfe-474f-b75e-aed67424afe1 from datanode DatanodeRegistration(127.0.0.1:39533, datanodeUuid=056dc2d8-52c9-4108-b42e-d3d4b78ab6f2, infoPort=34305, infoSecurePort=0, ipcPort=43961, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef4e373579d24a74 with lease ID 0x2b2881703bc15c: from storage DS-49fdbf28-adfe-474f-b75e-aed67424afe1 node DatanodeRegistration(127.0.0.1:39533, datanodeUuid=056dc2d8-52c9-4108-b42e-d3d4b78ab6f2, infoPort=34305, infoSecurePort=0, ipcPort=43961, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7abb297e25a0b74 with lease ID 0x2b2881703bc15d: Processing first storage report for DS-36564453-9851-48c6-a9b0-c611f3604e68 from datanode DatanodeRegistration(127.0.0.1:38347, datanodeUuid=fb294b86-2cb6-45be-b146-b1a7befb00f0, infoPort=41233, infoSecurePort=0, ipcPort=41885, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7abb297e25a0b74 with lease ID 0x2b2881703bc15d: from storage DS-36564453-9851-48c6-a9b0-c611f3604e68 node DatanodeRegistration(127.0.0.1:38347, datanodeUuid=fb294b86-2cb6-45be-b146-b1a7befb00f0, infoPort=41233, infoSecurePort=0, ipcPort=41885, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8b5cb8657a859c77 with lease ID 0x2b2881703bc15e: Processing first storage report for DS-9c00ce78-49c7-4d2d-8cd5-771a2adb111a from datanode DatanodeRegistration(127.0.0.1:46093, datanodeUuid=d1970de4-eae2-4112-a0d1-f0d815042ff4, infoPort=34509, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661) 2024-11-20T10:30:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8b5cb8657a859c77 with lease ID 0x2b2881703bc15e: from storage DS-9c00ce78-49c7-4d2d-8cd5-771a2adb111a node DatanodeRegistration(127.0.0.1:46093, datanodeUuid=d1970de4-eae2-4112-a0d1-f0d815042ff4, infoPort=34509, infoSecurePort=0, ipcPort=36761, storageInfo=lv=-57;cid=testClusterID;nsid=1287233170;c=1732098649661), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T10:30:53,519 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d 2024-11-20T10:30:53,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/zookeeper_0, clientPort=63828, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T10:30:53,608 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63828 2024-11-20T10:30:53,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:53,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:53,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741825_1001 (size=7) 2024-11-20T10:30:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741825_1001 (size=7) 2024-11-20T10:30:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741825_1001 (size=7) 2024-11-20T10:30:54,195 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 with version=8 2024-11-20T10:30:54,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/hbase-staging 2024-11-20T10:30:54,481 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:30:54,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:54,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:54,495 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:30:54,495 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:54,495 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:30:54,626 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T10:30:54,682 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T10:30:54,692 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T10:30:54,695 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:30:54,720 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90239 (auto-detected) 2024-11-20T10:30:54,721 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T10:30:54,741 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38325 2024-11-20T10:30:54,760 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38325 connecting to ZooKeeper ensemble=127.0.0.1:63828 2024-11-20T10:30:54,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383250x0, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:30:54,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38325-0x10157cbf93d0000 connected 2024-11-20T10:30:54,998 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,004 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:30:55,020 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87, hbase.cluster.distributed=false 2024-11-20T10:30:55,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:30:55,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38325 2024-11-20T10:30:55,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38325 2024-11-20T10:30:55,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38325 2024-11-20T10:30:55,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38325 2024-11-20T10:30:55,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38325 2024-11-20T10:30:55,140 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:30:55,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,142 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:30:55,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:30:55,145 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T10:30:55,148 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:30:55,149 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40989 2024-11-20T10:30:55,151 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40989 connecting to ZooKeeper ensemble=127.0.0.1:63828 2024-11-20T10:30:55,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,163 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409890x0, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:30:55,164 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40989-0x10157cbf93d0001 connected 2024-11-20T10:30:55,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:30:55,167 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T10:30:55,175 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T10:30:55,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T10:30:55,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:30:55,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-20T10:30:55,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40989 2024-11-20T10:30:55,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40989 2024-11-20T10:30:55,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-20T10:30:55,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40989 2024-11-20T10:30:55,203 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:30:55,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,204 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:30:55,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:30:55,204 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T10:30:55,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:30:55,205 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38107 2024-11-20T10:30:55,207 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38107 connecting to ZooKeeper ensemble=127.0.0.1:63828 2024-11-20T10:30:55,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,210 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381070x0, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:30:55,226 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381070x0, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:30:55,226 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38107-0x10157cbf93d0002 connected 2024-11-20T10:30:55,227 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T10:30:55,228 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T10:30:55,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T10:30:55,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:30:55,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38107 2024-11-20T10:30:55,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38107 2024-11-20T10:30:55,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38107 2024-11-20T10:30:55,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38107 2024-11-20T10:30:55,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38107 2024-11-20T10:30:55,250 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8c533afd331c:0 server-side Connection retries=45 2024-11-20T10:30:55,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,251 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T10:30:55,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T10:30:55,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T10:30:55,252 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T10:30:55,252 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T10:30:55,253 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41755 2024-11-20T10:30:55,255 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41755 connecting to ZooKeeper ensemble=127.0.0.1:63828 2024-11-20T10:30:55,257 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,259 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417550x0, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T10:30:55,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417550x0, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:30:55,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41755-0x10157cbf93d0003 connected 2024-11-20T10:30:55,270 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T10:30:55,271 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T10:30:55,272 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T10:30:55,274 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T10:30:55,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41755 2024-11-20T10:30:55,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41755 2024-11-20T10:30:55,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41755 2024-11-20T10:30:55,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41755 2024-11-20T10:30:55,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41755 2024-11-20T10:30:55,297 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8c533afd331c:38325 2024-11-20T10:30:55,298 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8c533afd331c,38325,1732098654332 2024-11-20T10:30:55,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,311 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,313 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8c533afd331c,38325,1732098654332 2024-11-20T10:30:55,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T10:30:55,342 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T10:30:55,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T10:30:55,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,342 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,343 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T10:30:55,344 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8c533afd331c,38325,1732098654332 from backup master directory 2024-11-20T10:30:55,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,353 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8c533afd331c,38325,1732098654332 2024-11-20T10:30:55,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T10:30:55,354 WARN [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:30:55,354 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8c533afd331c,38325,1732098654332 2024-11-20T10:30:55,357 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T10:30:55,358 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T10:30:55,414 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/hbase.id] with ID: 43f2778b-6f98-41d1-8c84-f7e0d018996c 2024-11-20T10:30:55,414 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/.tmp/hbase.id 2024-11-20T10:30:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741826_1002 (size=42) 2024-11-20T10:30:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741826_1002 (size=42) 2024-11-20T10:30:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741826_1002 (size=42) 2024-11-20T10:30:55,429 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/.tmp/hbase.id]:[hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/hbase.id] 2024-11-20T10:30:55,470 INFO [master/8c533afd331c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T10:30:55,476 INFO [master/8c533afd331c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T10:30:55,495 INFO [master/8c533afd331c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T10:30:55,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,513 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:55,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741827_1003 (size=196) 2024-11-20T10:30:55,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741827_1003 (size=196) 2024-11-20T10:30:55,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741827_1003 (size=196) 2024-11-20T10:30:55,546 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:30:55,548 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T10:30:55,553 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:55,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741828_1004 (size=1189) 2024-11-20T10:30:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741828_1004 (size=1189) 2024-11-20T10:30:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741828_1004 (size=1189) 2024-11-20T10:30:55,601 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store 2024-11-20T10:30:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741829_1005 (size=34) 2024-11-20T10:30:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741829_1005 (size=34) 2024-11-20T10:30:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741829_1005 (size=34) 2024-11-20T10:30:55,626 INFO [master/8c533afd331c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T10:30:55,629 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:55,630 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T10:30:55,630 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:30:55,630 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:30:55,631 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T10:30:55,631 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:30:55,632 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:30:55,632 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732098655630Disabling compacts and flushes for region at 1732098655630Disabling writes for close at 1732098655631 (+1 ms)Writing region close event to WAL at 1732098655631Closed at 1732098655631 2024-11-20T10:30:55,634 WARN [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/.initializing 2024-11-20T10:30:55,634 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/WALs/8c533afd331c,38325,1732098654332 2024-11-20T10:30:55,642 INFO [master/8c533afd331c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:55,655 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C38325%2C1732098654332, suffix=, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/WALs/8c533afd331c,38325,1732098654332, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/oldWALs, maxLogs=10 2024-11-20T10:30:55,681 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/WALs/8c533afd331c,38325,1732098654332/8c533afd331c%2C38325%2C1732098654332.1732098655659, exclude list is [], retry=0 2024-11-20T10:30:55,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:30:55,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:55,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:55,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:55,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T10:30:55,738 INFO [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/WALs/8c533afd331c,38325,1732098654332/8c533afd331c%2C38325%2C1732098654332.1732098655659 2024-11-20T10:30:55,739 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:55,739 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:30:55,740 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:55,743 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,743 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T10:30:55,805 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:55,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:55,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T10:30:55,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:55,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:55,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T10:30:55,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:55,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:55,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T10:30:55,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:55,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:55,820 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,823 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,824 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,829 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,829 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,832 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T10:30:55,835 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T10:30:55,839 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:30:55,840 INFO [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60717851, jitterRate=-0.09523351490497589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T10:30:55,848 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732098655758Initializing all the Stores at 1732098655760 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098655760Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098655761 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098655761Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098655761Cleaning up temporary data from old regions at 1732098655829 (+68 ms)Region opened successfully at 1732098655848 (+19 ms) 2024-11-20T10:30:55,849 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T10:30:55,879 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4ab40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:30:55,905 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T10:30:55,915 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T10:30:55,915 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T10:30:55,917 INFO [master/8c533afd331c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T10:30:55,918 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T10:30:55,923 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T10:30:55,923 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T10:30:55,945 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T10:30:55,953 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T10:30:56,005 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T10:30:56,009 INFO [master/8c533afd331c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T10:30:56,012 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T10:30:56,025 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T10:30:56,027 INFO [master/8c533afd331c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T10:30:56,030 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T10:30:56,036 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T10:30:56,038 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T10:30:56,047 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T10:30:56,064 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T10:30:56,077 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T10:30:56,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:30:56,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:30:56,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:30:56,089 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T10:30:56,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,090 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,094 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8c533afd331c,38325,1732098654332, sessionid=0x10157cbf93d0000, setting cluster-up flag (Was=false) 2024-11-20T10:30:56,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,121 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,152 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T10:30:56,156 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8c533afd331c,38325,1732098654332 2024-11-20T10:30:56,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,184 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,215 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T10:30:56,218 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8c533afd331c,38325,1732098654332 2024-11-20T10:30:56,227 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T10:30:56,281 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(746): ClusterId : 43f2778b-6f98-41d1-8c84-f7e0d018996c 2024-11-20T10:30:56,281 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(746): ClusterId : 43f2778b-6f98-41d1-8c84-f7e0d018996c 2024-11-20T10:30:56,281 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(746): ClusterId : 43f2778b-6f98-41d1-8c84-f7e0d018996c 2024-11-20T10:30:56,284 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T10:30:56,284 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T10:30:56,284 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T10:30:56,291 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T10:30:56,291 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T10:30:56,291 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T10:30:56,291 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T10:30:56,291 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T10:30:56,291 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T10:30:56,300 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T10:30:56,309 INFO [master/8c533afd331c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T10:30:56,311 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T10:30:56,311 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T10:30:56,311 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T10:30:56,312 DEBUG [RS:0;8c533afd331c:40989 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ef2963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:30:56,312 DEBUG [RS:2;8c533afd331c:41755 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e73e33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:30:56,312 DEBUG [RS:1;8c533afd331c:38107 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7001a4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8c533afd331c/172.17.0.2:0 2024-11-20T10:30:56,318 INFO [master/8c533afd331c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T10:30:56,326 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8c533afd331c:38107 2024-11-20T10:30:56,325 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8c533afd331c,38325,1732098654332 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T10:30:56,329 INFO [RS:1;8c533afd331c:38107 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T10:30:56,329 INFO [RS:1;8c533afd331c:38107 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T10:30:56,329 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T10:30:56,330 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;8c533afd331c:41755 2024-11-20T10:30:56,330 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8c533afd331c:40989 2024-11-20T10:30:56,331 INFO [RS:2;8c533afd331c:41755 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T10:30:56,331 INFO [RS:0;8c533afd331c:40989 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T10:30:56,331 INFO [RS:2;8c533afd331c:41755 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T10:30:56,331 INFO [RS:0;8c533afd331c:40989 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T10:30:56,331 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T10:30:56,331 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T10:30:56,332 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(2659): reportForDuty to master=8c533afd331c,38325,1732098654332 with port=38107, startcode=1732098655202 2024-11-20T10:30:56,332 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(2659): reportForDuty to master=8c533afd331c,38325,1732098654332 with port=41755, startcode=1732098655250 2024-11-20T10:30:56,332 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(2659): reportForDuty to master=8c533afd331c,38325,1732098654332 with port=40989, startcode=1732098655109 2024-11-20T10:30:56,333 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:30:56,333 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:30:56,333 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:30:56,334 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8c533afd331c:0, corePoolSize=5, maxPoolSize=5 2024-11-20T10:30:56,334 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8c533afd331c:0, corePoolSize=10, maxPoolSize=10 2024-11-20T10:30:56,334 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,334 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:30:56,334 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,339 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732098686339 2024-11-20T10:30:56,340 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T10:30:56,341 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T10:30:56,344 DEBUG [RS:2;8c533afd331c:41755 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T10:30:56,344 DEBUG [RS:0;8c533afd331c:40989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T10:30:56,344 DEBUG [RS:1;8c533afd331c:38107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T10:30:56,344 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:30:56,345 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T10:30:56,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T10:30:56,346 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T10:30:56,347 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T10:30:56,347 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T10:30:56,347 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,350 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:56,350 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T10:30:56,352 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T10:30:56,354 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T10:30:56,354 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T10:30:56,357 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T10:30:56,357 INFO [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T10:30:56,360 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098656358,5,FailOnTimeoutGroup] 2024-11-20T10:30:56,363 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098656360,5,FailOnTimeoutGroup] 2024-11-20T10:30:56,363 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,363 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T10:30:56,365 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,365 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741831_1007 (size=1321) 2024-11-20T10:30:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741831_1007 (size=1321) 2024-11-20T10:30:56,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741831_1007 (size=1321) 2024-11-20T10:30:56,395 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T10:30:56,396 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 2024-11-20T10:30:56,404 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51107, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T10:30:56,404 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T10:30:56,404 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T10:30:56,410 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,412 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(517): Registering regionserver=8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741832_1008 (size=32) 2024-11-20T10:30:56,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741832_1008 (size=32) 2024-11-20T10:30:56,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741832_1008 (size=32) 2024-11-20T10:30:56,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:56,423 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,424 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(517): Registering regionserver=8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T10:30:56,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T10:30:56,428 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 2024-11-20T10:30:56,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:56,428 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36923 2024-11-20T10:30:56,428 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T10:30:56,429 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,429 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] master.ServerManager(517): Registering regionserver=8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:56,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T10:30:56,432 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 2024-11-20T10:30:56,432 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36923 2024-11-20T10:30:56,432 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T10:30:56,434 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 2024-11-20T10:30:56,434 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36923 2024-11-20T10:30:56,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T10:30:56,434 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T10:30:56,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:56,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:56,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T10:30:56,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T10:30:56,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T10:30:56,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:56,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:56,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T10:30:56,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T10:30:56,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:56,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:56,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T10:30:56,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:30:56,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:30:56,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T10:30:56,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T10:30:56,451 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T10:30:56,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T10:30:56,459 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:30:56,460 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75405852, jitterRate=0.12363475561141968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T10:30:56,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732098656421Initializing all the Stores at 1732098656423 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098656424 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098656424Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098656424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098656424Cleaning up temporary data from old regions at 1732098656450 (+26 ms)Region opened successfully at 1732098656465 (+15 ms) 2024-11-20T10:30:56,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T10:30:56,465 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T10:30:56,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T10:30:56,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T10:30:56,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T10:30:56,467 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T10:30:56,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732098656465Disabling compacts and flushes for region at 1732098656465Disabling writes for close at 1732098656466 (+1 ms)Writing region close event to WAL at 1732098656467 (+1 ms)Closed at 1732098656467 2024-11-20T10:30:56,470 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:30:56,471 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T10:30:56,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T10:30:56,476 DEBUG [RS:1;8c533afd331c:38107 {}] zookeeper.ZKUtil(111): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,477 WARN [RS:1;8c533afd331c:38107 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:30:56,477 INFO [RS:1;8c533afd331c:38107 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:56,477 DEBUG [RS:0;8c533afd331c:40989 {}] zookeeper.ZKUtil(111): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,477 DEBUG [RS:2;8c533afd331c:41755 {}] zookeeper.ZKUtil(111): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,477 WARN [RS:0;8c533afd331c:40989 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:30:56,477 WARN [RS:2;8c533afd331c:41755 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T10:30:56,477 INFO [RS:0;8c533afd331c:40989 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:56,477 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,477 INFO [RS:2;8c533afd331c:41755 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:56,477 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,477 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8c533afd331c,41755,1732098655250] 2024-11-20T10:30:56,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8c533afd331c,40989,1732098655109] 2024-11-20T10:30:56,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8c533afd331c,38107,1732098655202] 2024-11-20T10:30:56,488 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:30:56,492 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T10:30:56,503 INFO [RS:0;8c533afd331c:40989 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T10:30:56,503 INFO [RS:1;8c533afd331c:38107 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T10:30:56,503 INFO [RS:2;8c533afd331c:41755 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T10:30:56,523 INFO [RS:1;8c533afd331c:38107 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T10:30:56,523 INFO [RS:0;8c533afd331c:40989 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T10:30:56,523 INFO [RS:2;8c533afd331c:41755 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T10:30:56,529 INFO [RS:0;8c533afd331c:40989 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:30:56,529 INFO [RS:2;8c533afd331c:41755 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:30:56,529 INFO [RS:1;8c533afd331c:38107 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:30:56,529 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,529 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,529 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,530 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T10:30:56,530 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T10:30:56,530 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T10:30:56,537 INFO [RS:0;8c533afd331c:40989 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T10:30:56,537 INFO [RS:1;8c533afd331c:38107 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T10:30:56,537 INFO [RS:2;8c533afd331c:41755 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T10:30:56,539 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,539 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,539 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,539 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:30:56,540 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,540 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:30:56,540 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0, corePoolSize=2, maxPoolSize=2 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:0;8c533afd331c:40989 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,541 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,541 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,542 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8c533afd331c:0, corePoolSize=1, maxPoolSize=1 2024-11-20T10:30:56,542 DEBUG [RS:1;8c533afd331c:38107 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,542 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,542 DEBUG [RS:2;8c533afd331c:41755 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8c533afd331c:0, corePoolSize=3, maxPoolSize=3 2024-11-20T10:30:56,545 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,545 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38107,1732098655202-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,40989,1732098655109-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,546 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,41755,1732098655250-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:30:56,574 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T10:30:56,574 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T10:30:56,574 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T10:30:56,576 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,40989,1732098655109-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,576 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38107,1732098655202-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,576 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,41755,1732098655250-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,577 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,577 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,577 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,577 INFO [RS:0;8c533afd331c:40989 {}] regionserver.Replication(171): 8c533afd331c,40989,1732098655109 started 2024-11-20T10:30:56,577 INFO [RS:2;8c533afd331c:41755 {}] regionserver.Replication(171): 8c533afd331c,41755,1732098655250 started 2024-11-20T10:30:56,577 INFO [RS:1;8c533afd331c:38107 {}] regionserver.Replication(171): 8c533afd331c,38107,1732098655202 started 2024-11-20T10:30:56,596 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,597 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1482): Serving as 8c533afd331c,38107,1732098655202, RpcServer on 8c533afd331c/172.17.0.2:38107, sessionid=0x10157cbf93d0002 2024-11-20T10:30:56,597 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T10:30:56,598 DEBUG [RS:1;8c533afd331c:38107 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,598 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,38107,1732098655202' 2024-11-20T10:30:56,598 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T10:30:56,599 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8c533afd331c,38107,1732098655202 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,38107,1732098655202' 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T10:30:56,600 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T10:30:56,600 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,600 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:56,601 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1482): Serving as 8c533afd331c,40989,1732098655109, RpcServer on 8c533afd331c/172.17.0.2:40989, sessionid=0x10157cbf93d0001 2024-11-20T10:30:56,601 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1482): Serving as 8c533afd331c,41755,1732098655250, RpcServer on 8c533afd331c/172.17.0.2:41755, sessionid=0x10157cbf93d0003 2024-11-20T10:30:56,601 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T10:30:56,601 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T10:30:56,601 DEBUG [RS:0;8c533afd331c:40989 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,601 DEBUG [RS:2;8c533afd331c:41755 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,601 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,41755,1732098655250' 2024-11-20T10:30:56,601 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,40989,1732098655109' 2024-11-20T10:30:56,601 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T10:30:56,601 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T10:30:56,601 DEBUG [RS:1;8c533afd331c:38107 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T10:30:56,601 INFO [RS:1;8c533afd331c:38107 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T10:30:56,601 INFO [RS:1;8c533afd331c:38107 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,41755,1732098655250' 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8c533afd331c,40989,1732098655109 2024-11-20T10:30:56,602 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8c533afd331c,40989,1732098655109' 2024-11-20T10:30:56,602 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T10:30:56,603 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T10:30:56,603 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T10:30:56,603 DEBUG [RS:2;8c533afd331c:41755 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T10:30:56,603 INFO [RS:2;8c533afd331c:41755 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T10:30:56,603 INFO [RS:2;8c533afd331c:41755 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T10:30:56,603 DEBUG [RS:0;8c533afd331c:40989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T10:30:56,603 INFO [RS:0;8c533afd331c:40989 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T10:30:56,604 INFO [RS:0;8c533afd331c:40989 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T10:30:56,643 WARN [8c533afd331c:38325 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T10:30:56,710 INFO [RS:0;8c533afd331c:40989 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:56,710 INFO [RS:1;8c533afd331c:38107 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:56,710 INFO [RS:2;8c533afd331c:41755 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:56,715 INFO [RS:0;8c533afd331c:40989 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C40989%2C1732098655109, suffix=, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,40989,1732098655109, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs, maxLogs=32 2024-11-20T10:30:56,715 INFO [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C38107%2C1732098655202, suffix=, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs, maxLogs=32 2024-11-20T10:30:56,715 INFO [RS:2;8c533afd331c:41755 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C41755%2C1732098655250, suffix=, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs, maxLogs=32 2024-11-20T10:30:56,731 DEBUG [RS:1;8c533afd331c:38107 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202/8c533afd331c%2C38107%2C1732098655202.1732098656718, exclude list is [], retry=0 2024-11-20T10:30:56,731 DEBUG [RS:2;8c533afd331c:41755 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250/8c533afd331c%2C41755%2C1732098655250.1732098656718, exclude list is [], retry=0 2024-11-20T10:30:56,732 DEBUG [RS:0;8c533afd331c:40989 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,40989,1732098655109/8c533afd331c%2C40989%2C1732098655109.1732098656718, exclude list is [], retry=0 2024-11-20T10:30:56,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:56,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:56,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:56,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:56,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:56,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:56,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:56,742 INFO [RS:0;8c533afd331c:40989 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,40989,1732098655109/8c533afd331c%2C40989%2C1732098655109.1732098656718 2024-11-20T10:30:56,742 INFO [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202/8c533afd331c%2C38107%2C1732098655202.1732098656718 2024-11-20T10:30:56,742 INFO [RS:2;8c533afd331c:41755 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250/8c533afd331c%2C41755%2C1732098655250.1732098656718 2024-11-20T10:30:56,742 DEBUG [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:30:56,742 DEBUG [RS:0;8c533afd331c:40989 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:56,742 DEBUG [RS:2;8c533afd331c:41755 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:56,897 DEBUG [8c533afd331c:38325 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T10:30:56,906 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(204): Hosts are {8c533afd331c=0} racks are {/default-rack=0} 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T10:30:56,912 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T10:30:56,912 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T10:30:56,912 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T10:30:56,912 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T10:30:56,918 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:30:56,925 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,41755,1732098655250, state=OPENING 2024-11-20T10:30:56,973 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T10:30:56,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,984 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:30:56,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:56,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:56,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:56,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:56,989 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:30:56,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,41755,1732098655250}] 2024-11-20T10:30:57,172 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:30:57,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41255, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:30:57,190 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T10:30:57,191 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:57,191 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T10:30:57,194 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C41755%2C1732098655250.meta, suffix=.meta, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs, maxLogs=32 2024-11-20T10:30:57,212 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, exclude list is [], retry=0 2024-11-20T10:30:57,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:57,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:57,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:57,218 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta 2024-11-20T10:30:57,219 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:57,219 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:30:57,221 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T10:30:57,223 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T10:30:57,228 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T10:30:57,232 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T10:30:57,232 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:57,233 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T10:30:57,233 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T10:30:57,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T10:30:57,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T10:30:57,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:57,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T10:30:57,239 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T10:30:57,239 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:57,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T10:30:57,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T10:30:57,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:57,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T10:30:57,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T10:30:57,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:30:57,245 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T10:30:57,246 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:30:57,249 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:30:57,252 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T10:30:57,252 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T10:30:57,253 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T10:30:57,255 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T10:30:57,257 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74818270, jitterRate=0.11487910151481628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T10:30:57,257 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T10:30:57,259 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732098657233Writing region info on filesystem at 1732098657233Initializing all the Stores at 1732098657235 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098657235Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098657235Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098657235Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098657235Cleaning up temporary data from old regions at 1732098657252 (+17 ms)Running coprocessor post-open hooks at 1732098657257 (+5 ms)Region opened successfully at 1732098657259 (+2 ms) 2024-11-20T10:30:57,267 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732098657161 2024-11-20T10:30:57,278 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T10:30:57,279 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T10:30:57,280 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:30:57,283 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,41755,1732098655250, state=OPEN 2024-11-20T10:30:57,289 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:30:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:30:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:30:57,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:30:57,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:57,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:57,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:57,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:30:57,290 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8c533afd331c,41755,1732098655250 2024-11-20T10:30:57,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T10:30:57,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,41755,1732098655250 in 299 msec 2024-11-20T10:30:57,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T10:30:57,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 821 msec 2024-11-20T10:30:57,303 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T10:30:57,303 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T10:30:57,340 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:30:57,342 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1] 2024-11-20T10:30:57,363 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:30:57,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33953, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:30:57,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1270 sec 2024-11-20T10:30:57,384 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732098657384, completionTime=-1 2024-11-20T10:30:57,387 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T10:30:57,389 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T10:30:57,411 INFO [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T10:30:57,411 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732098717411 2024-11-20T10:30:57,411 INFO [master/8c533afd331c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732098777411 2024-11-20T10:30:57,411 INFO [master/8c533afd331c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 22 msec 2024-11-20T10:30:57,412 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T10:30:57,418 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,418 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,418 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,420 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8c533afd331c:38325, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,420 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,420 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,425 DEBUG [master/8c533afd331c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T10:30:57,443 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.088sec 2024-11-20T10:30:57,445 INFO [master/8c533afd331c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T10:30:57,446 INFO [master/8c533afd331c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T10:30:57,446 INFO [master/8c533afd331c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T10:30:57,447 INFO [master/8c533afd331c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T10:30:57,447 INFO [master/8c533afd331c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T10:30:57,447 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T10:30:57,448 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T10:30:57,452 DEBUG [master/8c533afd331c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T10:30:57,452 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T10:30:57,453 INFO [master/8c533afd331c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8c533afd331c,38325,1732098654332-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T10:30:57,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ced8e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:30:57,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8c533afd331c,38325,-1 for getting cluster id 2024-11-20T10:30:57,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T10:30:57,504 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '43f2778b-6f98-41d1-8c84-f7e0d018996c' 2024-11-20T10:30:57,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T10:30:57,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "43f2778b-6f98-41d1-8c84-f7e0d018996c" 2024-11-20T10:30:57,507 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763eb49f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:30:57,507 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8c533afd331c,38325,-1] 2024-11-20T10:30:57,510 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T10:30:57,512 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:30:57,513 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T10:30:57,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d952677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T10:30:57,516 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:30:57,523 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1] 2024-11-20T10:30:57,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:30:57,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:30:57,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8c533afd331c,38325,1732098654332 2024-11-20T10:30:57,543 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:36923/hbase 2024-11-20T10:30:57,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=8612 2024-11-20T10:30:57,573 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:57,576 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:57,577 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:57,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71733545, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-71733545, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:57,596 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71733545/hregion-71733545.1732098657583, exclude list is [], retry=0 2024-11-20T10:30:57,600 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:57,600 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:57,601 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:57,604 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71733545/hregion-71733545.1732098657583 2024-11-20T10:30:57,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:30:57,605 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a6f1fa3fd54c57a33d434ed8c0c0557e, NAME => 'testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:30:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741838_1014 (size=64) 2024-11-20T10:30:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741838_1014 (size=64) 2024-11-20T10:30:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741838_1014 (size=64) 2024-11-20T10:30:57,620 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:57,622 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,625 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6f1fa3fd54c57a33d434ed8c0c0557e columnFamilyName a 2024-11-20T10:30:57,625 DEBUG [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,627 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] regionserver.HStore(327): Store=a6f1fa3fd54c57a33d434ed8c0c0557e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:57,627 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,629 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,630 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,630 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,633 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,637 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:30:57,638 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a6f1fa3fd54c57a33d434ed8c0c0557e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61524304, jitterRate=-0.08321642875671387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:30:57,640 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a6f1fa3fd54c57a33d434ed8c0c0557e: Writing region info on filesystem at 1732098657621Initializing all the Stores at 1732098657622 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098657622Cleaning up temporary data from old regions at 1732098657630 (+8 ms)Region opened successfully at 1732098657640 (+10 ms) 2024-11-20T10:30:57,640 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a6f1fa3fd54c57a33d434ed8c0c0557e, disabling compactions & flushes 2024-11-20T10:30:57,640 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. 2024-11-20T10:30:57,640 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. 2024-11-20T10:30:57,640 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. after waiting 0 ms 2024-11-20T10:30:57,640 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. 2024-11-20T10:30:57,641 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. 2024-11-20T10:30:57,641 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a6f1fa3fd54c57a33d434ed8c0c0557e: Waiting for close lock at 1732098657640Disabling compacts and flushes for region at 1732098657640Disabling writes for close at 1732098657640Writing region close event to WAL at 1732098657641 (+1 ms)Closed at 1732098657641 2024-11-20T10:30:57,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741837_1013 (size=93) 2024-11-20T10:30:57,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741837_1013 (size=93) 2024-11-20T10:30:57,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741837_1013 (size=93) 2024-11-20T10:30:57,652 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:30:57,652 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71733545:(num 1732098657583) 2024-11-20T10:30:57,654 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T10:30:57,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741839_1015 (size=369) 2024-11-20T10:30:57,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741839_1015 (size=369) 2024-11-20T10:30:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741839_1015 (size=369) 2024-11-20T10:30:57,672 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T10:30:57,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741840_1016 (size=231) 2024-11-20T10:30:57,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741840_1016 (size=231) 2024-11-20T10:30:57,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741840_1016 (size=231) 2024-11-20T10:30:57,705 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1, size=369 (369bytes) 2024-11-20T10:30:57,706 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T10:30:57,706 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T10:30:57,706 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1 2024-11-20T10:30:57,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1 after 3ms 2024-11-20T10:30:57,718 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1 took 14ms 2024-11-20T10:30:57,722 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1 so closing down 2024-11-20T10:30:57,725 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T10:30:57,727 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T10:30:57,727 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:30:57,728 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:30:57,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741841_1017 (size=369) 2024-11-20T10:30:57,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741841_1017 (size=369) 2024-11-20T10:30:57,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741841_1017 (size=369) 2024-11-20T10:30:57,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T10:30:57,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002 2024-11-20T10:30:57,748 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-20T10:30:57,748 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1, journal: Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1, size=369 (369bytes) at 1732098657705Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1 so closing down at 1732098657723 (+18 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp at 1732098657727 (+4 ms)3 split writer threads finished at 1732098657728 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732098657741 (+13 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002 at 1732098657743 (+2 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732098657748 (+5 ms) 2024-11-20T10:30:57,763 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2, size=231 (231bytes) 2024-11-20T10:30:57,763 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2 2024-11-20T10:30:57,764 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2 after 1ms 2024-11-20T10:30:57,767 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2 took 4ms 2024-11-20T10:30:57,769 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2 so closing down 2024-11-20T10:30:57,770 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:30:57,772 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T10:30:57,773 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T10:30:57,773 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:30:57,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741842_1018 (size=231) 2024-11-20T10:30:57,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741842_1018 (size=231) 2024-11-20T10:30:57,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741842_1018 (size=231) 2024-11-20T10:30:57,782 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T10:30:57,789 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-20T10:30:57,792 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-20T10:30:57,792 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2, journal: Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2, size=231 (231bytes) at 1732098657763Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2 so closing down at 1732098657769 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002-wal-2.temp at 1732098657773 (+4 ms)3 split writer threads finished at 1732098657773Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732098657782 (+9 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732098657792 (+10 ms) 2024-11-20T10:30:57,792 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:30:57,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:57,813 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal.1732098657796, exclude list is [], retry=0 2024-11-20T10:30:57,817 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:57,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:57,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:57,820 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal.1732098657796 2024-11-20T10:30:57,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a6f1fa3fd54c57a33d434ed8c0c0557e, NAME => 'testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:30:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,821 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,823 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,824 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6f1fa3fd54c57a33d434ed8c0c0557e columnFamilyName a 2024-11-20T10:30:57,824 DEBUG [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:57,825 INFO [StoreOpener-a6f1fa3fd54c57a33d434ed8c0c0557e-1 {}] regionserver.HStore(327): Store=a6f1fa3fd54c57a33d434ed8c0c0557e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:57,826 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,832 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002 2024-11-20T10:30:57,840 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002 2024-11-20T10:30:57,844 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a6f1fa3fd54c57a33d434ed8c0c0557e 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T10:30:57,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/.tmp/a/0edccced5b1e4251987960ce8577b226 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732098657652/Put/seqid=0 2024-11-20T10:30:57,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741844_1020 (size=5170) 2024-11-20T10:30:57,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741844_1020 (size=5170) 2024-11-20T10:30:57,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741844_1020 (size=5170) 2024-11-20T10:30:57,910 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/.tmp/a/0edccced5b1e4251987960ce8577b226 2024-11-20T10:30:57,951 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/.tmp/a/0edccced5b1e4251987960ce8577b226 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/a/0edccced5b1e4251987960ce8577b226 2024-11-20T10:30:57,962 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/a/0edccced5b1e4251987960ce8577b226, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T10:30:57,967 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a6f1fa3fd54c57a33d434ed8c0c0557e in 122ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T10:30:57,969 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/0000000000000000002 2024-11-20T10:30:57,969 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,969 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a6f1fa3fd54c57a33d434ed8c0c0557e 2024-11-20T10:30:57,976 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/a6f1fa3fd54c57a33d434ed8c0c0557e/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T10:30:57,977 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a6f1fa3fd54c57a33d434ed8c0c0557e; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63952939, jitterRate=-0.047026947140693665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:30:57,978 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a6f1fa3fd54c57a33d434ed8c0c0557e: Writing region info on filesystem at 1732098657821Initializing all the Stores at 1732098657823 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098657823Obtaining lock to block concurrent updates at 1732098657844 (+21 ms)Preparing flush snapshotting stores in a6f1fa3fd54c57a33d434ed8c0c0557e at 1732098657844Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732098657848 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732098657574.a6f1fa3fd54c57a33d434ed8c0c0557e. at 1732098657848Flushing a6f1fa3fd54c57a33d434ed8c0c0557e/a: creating writer at 1732098657850 (+2 ms)Flushing a6f1fa3fd54c57a33d434ed8c0c0557e/a: appending metadata at 1732098657885 (+35 ms)Flushing a6f1fa3fd54c57a33d434ed8c0c0557e/a: closing flushed file at 1732098657888 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4edde60c: reopening flushed file at 1732098657949 (+61 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a6f1fa3fd54c57a33d434ed8c0c0557e in 122ms, sequenceid=2, compaction requested=false; wal=null at 1732098657967 (+18 ms)Cleaning up temporary data from old regions at 1732098657969 (+2 ms)Region opened successfully at 1732098657978 (+9 ms) 2024-11-20T10:30:58,009 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 363) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36923/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35838 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:45602 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:36010 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:45712 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57326 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57520 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=8589 (was 8612) 2024-11-20T10:30:58,021 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=8587 2024-11-20T10:30:58,035 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:58,037 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:58,038 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:58,041 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15328591, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-15328591, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:58,054 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15328591/hregion-15328591.1732098658042, exclude list is [], retry=0 2024-11-20T10:30:58,058 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:58,058 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:58,058 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:58,061 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15328591/hregion-15328591.1732098658042 2024-11-20T10:30:58,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:30:58,062 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1a91389c67654466cdeb5e4c0663e766, NAME => 'testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:30:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741846_1022 (size=64) 2024-11-20T10:30:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741846_1022 (size=64) 2024-11-20T10:30:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741846_1022 (size=64) 2024-11-20T10:30:58,076 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:58,078 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,080 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91389c67654466cdeb5e4c0663e766 columnFamilyName a 2024-11-20T10:30:58,080 DEBUG [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:58,081 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] regionserver.HStore(327): Store=1a91389c67654466cdeb5e4c0663e766/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:58,081 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,083 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,083 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,085 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,088 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:30:58,089 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a91389c67654466cdeb5e4c0663e766; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65430751, jitterRate=-0.025005832314491272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:30:58,089 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a91389c67654466cdeb5e4c0663e766: Writing region info on filesystem at 1732098658076Initializing all the Stores at 1732098658078 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098658078Cleaning up temporary data from old regions at 1732098658083 (+5 ms)Region opened successfully at 1732098658089 (+6 ms) 2024-11-20T10:30:58,089 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1a91389c67654466cdeb5e4c0663e766, disabling compactions & flushes 2024-11-20T10:30:58,089 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. 2024-11-20T10:30:58,089 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. 2024-11-20T10:30:58,089 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. after waiting 0 ms 2024-11-20T10:30:58,089 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. 2024-11-20T10:30:58,090 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. 2024-11-20T10:30:58,090 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1a91389c67654466cdeb5e4c0663e766: Waiting for close lock at 1732098658089Disabling compacts and flushes for region at 1732098658089Disabling writes for close at 1732098658089Writing region close event to WAL at 1732098658090 (+1 ms)Closed at 1732098658090 2024-11-20T10:30:58,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741845_1021 (size=93) 2024-11-20T10:30:58,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741845_1021 (size=93) 2024-11-20T10:30:58,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741845_1021 (size=93) 2024-11-20T10:30:58,096 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:30:58,096 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15328591:(num 1732098658042) 2024-11-20T10:30:58,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T10:30:58,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741847_1023 (size=369) 2024-11-20T10:30:58,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741847_1023 (size=369) 2024-11-20T10:30:58,108 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T10:30:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741847_1023 (size=369) 2024-11-20T10:30:58,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741848_1024 (size=231) 2024-11-20T10:30:58,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741848_1024 (size=231) 2024-11-20T10:30:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741848_1024 (size=231) 2024-11-20T10:30:58,136 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2, size=231 (231bytes) 2024-11-20T10:30:58,136 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2 2024-11-20T10:30:58,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2 after 1ms 2024-11-20T10:30:58,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2 took 5ms 2024-11-20T10:30:58,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2 so closing down 2024-11-20T10:30:58,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:30:58,145 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T10:30:58,147 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T10:30:58,147 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:30:58,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741849_1025 (size=231) 2024-11-20T10:30:58,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741849_1025 (size=231) 2024-11-20T10:30:58,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741849_1025 (size=231) 2024-11-20T10:30:58,156 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T10:30:58,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 2024-11-20T10:30:58,158 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-20T10:30:58,158 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2, journal: Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2, size=231 (231bytes) at 1732098658136Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2 so closing down at 1732098658143 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp at 1732098658147 (+4 ms)3 split writer threads finished at 1732098658147Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732098658156 (+9 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 at 1732098658157 (+1 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732098658158 (+1 ms) 2024-11-20T10:30:58,172 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1, size=369 (369bytes) 2024-11-20T10:30:58,172 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1 2024-11-20T10:30:58,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1 after 1ms 2024-11-20T10:30:58,176 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1 took 4ms 2024-11-20T10:30:58,178 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1 so closing down 2024-11-20T10:30:58,178 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:30:58,181 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T10:30:58,182 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T10:30:58,182 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:30:58,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741850_1026 (size=369) 2024-11-20T10:30:58,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741850_1026 (size=369) 2024-11-20T10:30:58,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741850_1026 (size=369) 2024-11-20T10:30:58,190 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T10:30:58,195 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002, length=231 2024-11-20T10:30:58,197 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 2024-11-20T10:30:58,197 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-20T10:30:58,197 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1, journal: Splitting hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1, size=369 (369bytes) at 1732098658172Finishing writing output for hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1 so closing down at 1732098658178 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp at 1732098658182 (+4 ms)3 split writer threads finished at 1732098658182Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732098658190 (+8 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 at 1732098658197 (+7 ms)Processed 2 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732098658197 2024-11-20T10:30:58,197 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:30:58,199 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:58,213 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal.1732098658200, exclude list is [], retry=0 2024-11-20T10:30:58,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:58,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:58,217 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:58,219 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal.1732098658200 2024-11-20T10:30:58,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:30:58,219 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a91389c67654466cdeb5e4c0663e766, NAME => 'testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:30:58,219 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:58,219 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,219 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,222 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,224 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a91389c67654466cdeb5e4c0663e766 columnFamilyName a 2024-11-20T10:30:58,224 DEBUG [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:58,225 INFO [StoreOpener-1a91389c67654466cdeb5e4c0663e766-1 {}] regionserver.HStore(327): Store=1a91389c67654466cdeb5e4c0663e766/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:58,225 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,229 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 2024-11-20T10:30:58,234 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 2024-11-20T10:30:58,234 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a91389c67654466cdeb5e4c0663e766 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T10:30:58,253 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/.tmp/a/f61a76b9b51d474eac339839210c936b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732098658096/Put/seqid=0 2024-11-20T10:30:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741852_1028 (size=5170) 2024-11-20T10:30:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741852_1028 (size=5170) 2024-11-20T10:30:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741852_1028 (size=5170) 2024-11-20T10:30:58,262 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/.tmp/a/f61a76b9b51d474eac339839210c936b 2024-11-20T10:30:58,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/.tmp/a/f61a76b9b51d474eac339839210c936b as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/a/f61a76b9b51d474eac339839210c936b 2024-11-20T10:30:58,281 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/a/f61a76b9b51d474eac339839210c936b, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T10:30:58,281 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a91389c67654466cdeb5e4c0663e766 in 47ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T10:30:58,282 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/0000000000000000002 2024-11-20T10:30:58,283 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,283 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,286 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a91389c67654466cdeb5e4c0663e766 2024-11-20T10:30:58,288 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/1a91389c67654466cdeb5e4c0663e766/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T10:30:58,290 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a91389c67654466cdeb5e4c0663e766; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68076041, jitterRate=0.014412060379981995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:30:58,290 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a91389c67654466cdeb5e4c0663e766: Writing region info on filesystem at 1732098658220Initializing all the Stores at 1732098658221 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098658221Obtaining lock to block concurrent updates at 1732098658234 (+13 ms)Preparing flush snapshotting stores in 1a91389c67654466cdeb5e4c0663e766 at 1732098658234Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732098658234Flushing stores of testReplayEditsWrittenIntoWAL,,1732098658036.1a91389c67654466cdeb5e4c0663e766. at 1732098658234Flushing 1a91389c67654466cdeb5e4c0663e766/a: creating writer at 1732098658234Flushing 1a91389c67654466cdeb5e4c0663e766/a: appending metadata at 1732098658252 (+18 ms)Flushing 1a91389c67654466cdeb5e4c0663e766/a: closing flushed file at 1732098658252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34e6496: reopening flushed file at 1732098658269 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a91389c67654466cdeb5e4c0663e766 in 47ms, sequenceid=2, compaction requested=false; wal=null at 1732098658281 (+12 ms)Cleaning up temporary data from old regions at 1732098658283 (+2 ms)Region opened successfully at 1732098658290 (+7 ms) 2024-11-20T10:30:58,306 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:36092 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35838 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:45602 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57584 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57326 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:45814 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=8582 (was 8587) 2024-11-20T10:30:58,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=8581 2024-11-20T10:30:58,333 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:58,335 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:30:58,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:30:58,338 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27669928, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-27669928, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:58,351 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27669928/hregion-27669928.1732098658339, exclude list is [], retry=0 2024-11-20T10:30:58,355 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:58,355 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:58,355 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:58,358 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27669928/hregion-27669928.1732098658339 2024-11-20T10:30:58,358 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:30:58,359 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4ed14314ac4dc5f4cc40d3389e624dbc, NAME => 'testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:30:58,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741854_1030 (size=64) 2024-11-20T10:30:58,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741854_1030 (size=64) 2024-11-20T10:30:58,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741854_1030 (size=64) 2024-11-20T10:30:58,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:30:58,374 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,376 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName a 2024-11-20T10:30:58,376 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:58,377 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:58,377 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,379 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName b 2024-11-20T10:30:58,379 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:58,380 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:58,380 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,382 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName c 2024-11-20T10:30:58,382 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:30:58,382 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:30:58,383 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,385 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,385 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,386 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:30:58,387 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:30:58,390 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:30:58,390 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ed14314ac4dc5f4cc40d3389e624dbc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64703905, jitterRate=-0.035836681723594666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:30:58,391 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ed14314ac4dc5f4cc40d3389e624dbc: Writing region info on filesystem at 1732098658371Initializing all the Stores at 1732098658372 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098658372Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098658374 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098658374Cleaning up temporary data from old regions at 1732098658385 (+11 ms)Region opened successfully at 1732098658391 (+6 ms) 2024-11-20T10:30:58,391 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ed14314ac4dc5f4cc40d3389e624dbc, disabling compactions & flushes 2024-11-20T10:30:58,391 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:30:58,391 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:30:58,391 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. after waiting 0 ms 2024-11-20T10:30:58,391 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:30:58,392 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:30:58,392 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ed14314ac4dc5f4cc40d3389e624dbc: Waiting for close lock at 1732098658391Disabling compacts and flushes for region at 1732098658391Disabling writes for close at 1732098658391Writing region close event to WAL at 1732098658391Closed at 1732098658391 2024-11-20T10:30:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741853_1029 (size=93) 2024-11-20T10:30:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741853_1029 (size=93) 2024-11-20T10:30:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741853_1029 (size=93) 2024-11-20T10:30:58,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:30:58,398 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27669928:(num 1732098658339) 2024-11-20T10:30:58,398 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:30:58,400 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:30:58,412 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, exclude list is [], retry=0 2024-11-20T10:30:58,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:30:58,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:30:58,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:30:58,419 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 2024-11-20T10:30:58,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:30:58,642 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, size=0 (0bytes) 2024-11-20T10:30:58,642 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 might be still open, length is 0 2024-11-20T10:30:58,642 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 2024-11-20T10:30:58,643 WARN [IPC Server handler 1 on default port 36923 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-20T10:30:58,644 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 after 2ms 2024-11-20T10:30:59,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57618 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46093:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57618 dst: /127.0.0.1:46093 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46093 remote=/127.0.0.1:57618]. Total timeout mills is 60000, 59187 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:30:59,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:45860 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45860 dst: /127.0.0.1:38347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:30:59,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:36128 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36128 dst: /127.0.0.1:39533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:30:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741855_1032 (size=470675) 2024-11-20T10:30:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741855_1032 (size=470675) 2024-11-20T10:30:59,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741855_1032 (size=470675) 2024-11-20T10:31:02,645 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 after 4003ms 2024-11-20T10:31:02,655 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 took 4012ms 2024-11-20T10:31:02,660 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732098658401.temp 2024-11-20T10:31:02,661 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp 2024-11-20T10:31:02,736 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401; continuing. 2024-11-20T10:31:02,736 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 so closing down 2024-11-20T10:31:02,736 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:02,736 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:02,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741856_1033 (size=470683) 2024-11-20T10:31:02,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741856_1033 (size=470683) 2024-11-20T10:31:02,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741856_1033 (size=470683) 2024-11-20T10:31:02,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp (wrote 3002 edits, skipped 0 edits in 28 ms) 2024-11-20T10:31:02,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 2024-11-20T10:31:02,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T10:31:02,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, journal: Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, size=0 (0bytes) at 1732098658642Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp at 1732098662661 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1732098662703 (+42 ms)Split 2048 edits, skipped 0 edits. at 1732098662720 (+17 ms)Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 so closing down at 1732098662736 (+16 ms)3 split writer threads finished at 1732098662736Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp (wrote 3002 edits, skipped 0 edits in 28 ms) at 1732098662741 (+5 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000000001-wal.1732098658401.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 at 1732098662743 (+2 ms)Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401, size=0, length=0, corrupted=false, cancelled=false at 1732098662743 2024-11-20T10:31:02,745 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098658401 2024-11-20T10:31:02,747 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 2024-11-20T10:31:02,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:02,747 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:31:02,750 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:02,769 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098662751, exclude list is [], retry=0 2024-11-20T10:31:02,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:02,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:02,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:02,776 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098662751 2024-11-20T10:31:02,776 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:31:02,777 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:02,778 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,780 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName a 2024-11-20T10:31:02,780 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:02,780 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:02,780 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,782 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName b 2024-11-20T10:31:02,782 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:02,782 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:02,783 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,784 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed14314ac4dc5f4cc40d3389e624dbc columnFamilyName c 2024-11-20T10:31:02,784 DEBUG [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:02,785 INFO [StoreOpener-4ed14314ac4dc5f4cc40d3389e624dbc-1 {}] regionserver.HStore(327): Store=4ed14314ac4dc5f4cc40d3389e624dbc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:02,785 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,786 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,788 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:02,789 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 2024-11-20T10:31:02,820 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T10:31:02,832 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T10:31:03,172 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ed14314ac4dc5f4cc40d3389e624dbc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T10:31:03,211 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/7633a37265b74746835df9942d6898b4 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732098658428/Put/seqid=0 2024-11-20T10:31:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741858_1035 (size=50463) 2024-11-20T10:31:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741858_1035 (size=50463) 2024-11-20T10:31:03,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741858_1035 (size=50463) 2024-11-20T10:31:03,219 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/7633a37265b74746835df9942d6898b4 2024-11-20T10:31:03,228 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/7633a37265b74746835df9942d6898b4 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/a/7633a37265b74746835df9942d6898b4 2024-11-20T10:31:03,236 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/a/7633a37265b74746835df9942d6898b4, entries=754, sequenceid=754, filesize=49.3 K 2024-11-20T10:31:03,236 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 4ed14314ac4dc5f4cc40d3389e624dbc in 65ms, sequenceid=754, compaction requested=false; wal=null 2024-11-20T10:31:03,257 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T10:31:03,257 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ed14314ac4dc5f4cc40d3389e624dbc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T10:31:03,266 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/91566776715041c481224501e159fe45 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732098658478/Put/seqid=0 2024-11-20T10:31:03,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741859_1036 (size=20072) 2024-11-20T10:31:03,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741859_1036 (size=20072) 2024-11-20T10:31:03,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741859_1036 (size=20072) 2024-11-20T10:31:03,275 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/91566776715041c481224501e159fe45 2024-11-20T10:31:03,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/fc4b60753ef14c5f9779aa34524cbecb is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732098658503/Put/seqid=0 2024-11-20T10:31:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741860_1037 (size=35835) 2024-11-20T10:31:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741860_1037 (size=35835) 2024-11-20T10:31:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741860_1037 (size=35835) 2024-11-20T10:31:03,314 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/fc4b60753ef14c5f9779aa34524cbecb 2024-11-20T10:31:03,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/a/91566776715041c481224501e159fe45 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/a/91566776715041c481224501e159fe45 2024-11-20T10:31:03,329 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/a/91566776715041c481224501e159fe45, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-20T10:31:03,330 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/fc4b60753ef14c5f9779aa34524cbecb as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/b/fc4b60753ef14c5f9779aa34524cbecb 2024-11-20T10:31:03,337 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/b/fc4b60753ef14c5f9779aa34524cbecb, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-20T10:31:03,337 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4ed14314ac4dc5f4cc40d3389e624dbc in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-20T10:31:03,345 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T10:31:03,346 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ed14314ac4dc5f4cc40d3389e624dbc 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T10:31:03,352 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/77be5e823bc8435b83b7781884c999c9 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732098658531/Put/seqid=0 2024-11-20T10:31:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741861_1038 (size=35082) 2024-11-20T10:31:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741861_1038 (size=35082) 2024-11-20T10:31:03,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741861_1038 (size=35082) 2024-11-20T10:31:03,361 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/77be5e823bc8435b83b7781884c999c9 2024-11-20T10:31:03,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/a98e210c74ad42f59555f320c3cecf20 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732098658571/Put/seqid=0 2024-11-20T10:31:03,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741862_1039 (size=20825) 2024-11-20T10:31:03,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741862_1039 (size=20825) 2024-11-20T10:31:03,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741862_1039 (size=20825) 2024-11-20T10:31:03,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/a98e210c74ad42f59555f320c3cecf20 2024-11-20T10:31:03,400 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/b/77be5e823bc8435b83b7781884c999c9 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/b/77be5e823bc8435b83b7781884c999c9 2024-11-20T10:31:03,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/b/77be5e823bc8435b83b7781884c999c9, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-20T10:31:03,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/a98e210c74ad42f59555f320c3cecf20 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/c/a98e210c74ad42f59555f320c3cecf20 2024-11-20T10:31:03,415 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/c/a98e210c74ad42f59555f320c3cecf20, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-20T10:31:03,415 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4ed14314ac4dc5f4cc40d3389e624dbc in 70ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-20T10:31:03,428 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732098658609/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:31:03,432 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 2024-11-20T10:31:03,433 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T10:31:03,433 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ed14314ac4dc5f4cc40d3389e624dbc 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-20T10:31:03,444 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/2942011f18c2454baf5437f4b358a669 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732098658578/Put/seqid=0 2024-11-20T10:31:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741863_1040 (size=50301) 2024-11-20T10:31:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741863_1040 (size=50301) 2024-11-20T10:31:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741863_1040 (size=50301) 2024-11-20T10:31:03,456 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/2942011f18c2454baf5437f4b358a669 2024-11-20T10:31:03,464 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2942011f18c2454baf5437f4b358a669 2024-11-20T10:31:03,467 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/.tmp/c/2942011f18c2454baf5437f4b358a669 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/c/2942011f18c2454baf5437f4b358a669 2024-11-20T10:31:03,473 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2942011f18c2454baf5437f4b358a669 2024-11-20T10:31:03,474 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/c/2942011f18c2454baf5437f4b358a669, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-20T10:31:03,474 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 4ed14314ac4dc5f4cc40d3389e624dbc in 41ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-20T10:31:03,475 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/0000000000000003002 2024-11-20T10:31:03,476 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:03,476 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:03,476 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T10:31:03,478 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ed14314ac4dc5f4cc40d3389e624dbc 2024-11-20T10:31:03,482 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenIntoWAL/4ed14314ac4dc5f4cc40d3389e624dbc/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-20T10:31:03,483 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ed14314ac4dc5f4cc40d3389e624dbc; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64149801, jitterRate=-0.04409347474575043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T10:31:03,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ed14314ac4dc5f4cc40d3389e624dbc: Writing region info on filesystem at 1732098662777Initializing all the Stores at 1732098662778 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098662778Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098662778Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098662778Cleaning up temporary data from old regions at 1732098663476 (+698 ms)Region opened successfully at 1732098663483 (+7 ms) 2024-11-20T10:31:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ed14314ac4dc5f4cc40d3389e624dbc, disabling compactions & flushes 2024-11-20T10:31:03,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:31:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:31:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. after waiting 0 ms 2024-11-20T10:31:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:31:03,559 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732098658333.4ed14314ac4dc5f4cc40d3389e624dbc. 2024-11-20T10:31:03,559 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ed14314ac4dc5f4cc40d3389e624dbc: Waiting for close lock at 1732098663557Disabling compacts and flushes for region at 1732098663557Disabling writes for close at 1732098663557Writing region close event to WAL at 1732098663559 (+2 ms)Closed at 1732098663559 2024-11-20T10:31:03,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741857_1034 (size=93) 2024-11-20T10:31:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741857_1034 (size=93) 2024-11-20T10:31:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741857_1034 (size=93) 2024-11-20T10:31:03,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:03,566 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732098662751) 2024-11-20T10:31:03,581 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=397 (was 384) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36923 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-401821784_22 at /127.0.0.1:35008 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:43961 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-401821784_22 at /127.0.0.1:50266 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41885 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:41885 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-401821784_22 at /127.0.0.1:57802 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43961 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:36923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=861 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=8531 (was 8581) 2024-11-20T10:31:03,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=397, OpenFileDescriptor=861, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=8530 2024-11-20T10:31:03,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:03,620 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:03,621 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:03,624 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11941219, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-11941219, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:03,642 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11941219/hregion-11941219.1732098663625, exclude list is [], retry=0 2024-11-20T10:31:03,645 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:03,645 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:03,645 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:03,648 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11941219/hregion-11941219.1732098663625 2024-11-20T10:31:03,648 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:03,648 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ec5fd4cececb596c024a28f5c8244da2, NAME => 'test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:03,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741865_1042 (size=43) 2024-11-20T10:31:03,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741865_1042 (size=43) 2024-11-20T10:31:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741865_1042 (size=43) 2024-11-20T10:31:03,661 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:03,664 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,666 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName a 2024-11-20T10:31:03,666 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:03,667 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:03,667 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,669 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName b 2024-11-20T10:31:03,669 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:03,670 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:03,670 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,672 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName c 2024-11-20T10:31:03,672 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:03,673 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:03,673 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,675 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,675 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,676 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:03,677 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:03,679 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:03,680 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec5fd4cececb596c024a28f5c8244da2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71014416, jitterRate=0.0581972599029541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec5fd4cececb596c024a28f5c8244da2: Writing region info on filesystem at 1732098663661Initializing all the Stores at 1732098663662 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098663662Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098663664 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098663664Cleaning up temporary data from old regions at 1732098663675 (+11 ms)Region opened successfully at 1732098663681 (+6 ms) 2024-11-20T10:31:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec5fd4cececb596c024a28f5c8244da2, disabling compactions & flushes 2024-11-20T10:31:03,681 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. after waiting 0 ms 2024-11-20T10:31:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:03,682 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:03,682 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec5fd4cececb596c024a28f5c8244da2: Waiting for close lock at 1732098663681Disabling compacts and flushes for region at 1732098663681Disabling writes for close at 1732098663681Writing region close event to WAL at 1732098663682 (+1 ms)Closed at 1732098663682 2024-11-20T10:31:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741864_1041 (size=93) 2024-11-20T10:31:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741864_1041 (size=93) 2024-11-20T10:31:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741864_1041 (size=93) 2024-11-20T10:31:03,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:03,688 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11941219:(num 1732098663625) 2024-11-20T10:31:03,688 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:03,691 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:03,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, exclude list is [], retry=0 2024-11-20T10:31:03,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:03,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:03,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:03,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 2024-11-20T10:31:03,713 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:03,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741866_1043 (size=344306) 2024-11-20T10:31:03,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741866_1043 (size=344306) 2024-11-20T10:31:03,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741866_1043 (size=344306) 2024-11-20T10:31:03,891 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, size=336.2 K (344306bytes) 2024-11-20T10:31:03,891 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 2024-11-20T10:31:03,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 after 1ms 2024-11-20T10:31:03,896 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 took 5ms 2024-11-20T10:31:03,901 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732098663691.temp 2024-11-20T10:31:03,903 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp 2024-11-20T10:31:03,929 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 so closing down 2024-11-20T10:31:03,929 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:03,929 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:03,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741867_1044 (size=344306) 2024-11-20T10:31:03,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741867_1044 (size=344306) 2024-11-20T10:31:03,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741867_1044 (size=344306) 2024-11-20T10:31:03,933 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-11-20T10:31:03,935 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp to hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 2024-11-20T10:31:03,935 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-20T10:31:03,935 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, journal: Splitting hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, size=336.2 K (344306bytes) at 1732098663891Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp at 1732098663903 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732098663907 (+4 ms)Split 2048 edits, skipped 0 edits. at 1732098663919 (+12 ms)Finishing writing output for hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 so closing down at 1732098663929 (+10 ms)3 split writer threads finished at 1732098663929Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1732098663933 (+4 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000000001-wal.1732098663691.temp to hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 at 1732098663935 (+2 ms)Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732098663935 2024-11-20T10:31:03,937 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663691 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098663691 2024-11-20T10:31:03,938 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 2024-11-20T10:31:03,938 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:03,940 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:03,953 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, exclude list is [], retry=0 2024-11-20T10:31:03,956 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:03,957 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:03,957 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:03,959 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 2024-11-20T10:31:03,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:04,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741868_1045 (size=344433) 2024-11-20T10:31:04,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741868_1045 (size=344433) 2024-11-20T10:31:04,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741868_1045 (size=344433) 2024-11-20T10:31:04,150 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, size=336.4 K (344433bytes) 2024-11-20T10:31:04,150 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 2024-11-20T10:31:04,151 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 after 1ms 2024-11-20T10:31:04,155 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 took 4ms 2024-11-20T10:31:04,158 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732098663941.temp 2024-11-20T10:31:04,160 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp 2024-11-20T10:31:04,176 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 so closing down 2024-11-20T10:31:04,176 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:04,176 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:04,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741869_1046 (size=344433) 2024-11-20T10:31:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741869_1046 (size=344433) 2024-11-20T10:31:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741869_1046 (size=344433) 2024-11-20T10:31:04,189 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp (wrote 3000 edits, skipped 0 edits in 7 ms) 2024-11-20T10:31:04,190 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp to hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 2024-11-20T10:31:04,190 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-20T10:31:04,190 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, journal: Splitting hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, size=336.4 K (344433bytes) at 1732098664150Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp at 1732098664161 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732098664163 (+2 ms)Split 2048 edits, skipped 0 edits. at 1732098664169 (+6 ms)Finishing writing output for hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 so closing down at 1732098664176 (+7 ms)3 split writer threads finished at 1732098664176Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp (wrote 3000 edits, skipped 0 edits in 7 ms) at 1732098664189 (+13 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003001-wal.1732098663941.temp to hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 at 1732098664190 (+1 ms)Processed 3000 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732098664190 2024-11-20T10:31:04,192 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098663941 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098663941 2024-11-20T10:31:04,193 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 2024-11-20T10:31:04,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:04,195 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/test2727-manual,16010,1732098663616, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:04,209 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098664195, exclude list is [], retry=0 2024-11-20T10:31:04,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:04,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:04,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:04,214 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098664195 2024-11-20T10:31:04,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:04,215 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec5fd4cececb596c024a28f5c8244da2, NAME => 'test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:04,215 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:04,215 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,215 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,217 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,218 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName a 2024-11-20T10:31:04,218 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:04,218 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:04,219 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,220 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName b 2024-11-20T10:31:04,220 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:04,221 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:04,221 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,222 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec5fd4cececb596c024a28f5c8244da2 columnFamilyName c 2024-11-20T10:31:04,222 DEBUG [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:04,223 INFO [StoreOpener-ec5fd4cececb596c024a28f5c8244da2-1 {}] regionserver.HStore(327): Store=ec5fd4cececb596c024a28f5c8244da2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:04,223 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,227 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 2024-11-20T10:31:04,265 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 2024-11-20T10:31:04,266 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 2024-11-20T10:31:04,301 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 2024-11-20T10:31:04,301 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec5fd4cececb596c024a28f5c8244da2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-20T10:31:04,321 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/a/6c0a1778f27644efb328c372fe198fde is 41, key is test2727/a:100/1732098663964/Put/seqid=0 2024-11-20T10:31:04,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741871_1048 (size=84227) 2024-11-20T10:31:04,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741871_1048 (size=84227) 2024-11-20T10:31:04,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741871_1048 (size=84227) 2024-11-20T10:31:04,329 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/a/6c0a1778f27644efb328c372fe198fde 2024-11-20T10:31:04,354 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/b/f9bcf8a84c76483bb947b03a2b63169b is 41, key is test2727/b:100/1732098664013/Put/seqid=0 2024-11-20T10:31:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741872_1049 (size=84609) 2024-11-20T10:31:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741872_1049 (size=84609) 2024-11-20T10:31:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741872_1049 (size=84609) 2024-11-20T10:31:04,360 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/b/f9bcf8a84c76483bb947b03a2b63169b 2024-11-20T10:31:04,392 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/c/52c5140c4e064250b15bdb7284731734 is 41, key is test2727/c:100/1732098664083/Put/seqid=0 2024-11-20T10:31:04,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741873_1050 (size=84609) 2024-11-20T10:31:04,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741873_1050 (size=84609) 2024-11-20T10:31:04,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741873_1050 (size=84609) 2024-11-20T10:31:04,398 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/c/52c5140c4e064250b15bdb7284731734 2024-11-20T10:31:04,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/a/6c0a1778f27644efb328c372fe198fde as hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/a/6c0a1778f27644efb328c372fe198fde 2024-11-20T10:31:04,415 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/a/6c0a1778f27644efb328c372fe198fde, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-20T10:31:04,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/b/f9bcf8a84c76483bb947b03a2b63169b as hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/b/f9bcf8a84c76483bb947b03a2b63169b 2024-11-20T10:31:04,424 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/b/f9bcf8a84c76483bb947b03a2b63169b, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T10:31:04,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/.tmp/c/52c5140c4e064250b15bdb7284731734 as hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/c/52c5140c4e064250b15bdb7284731734 2024-11-20T10:31:04,433 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/c/52c5140c4e064250b15bdb7284731734, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T10:31:04,433 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ec5fd4cececb596c024a28f5c8244da2 in 132ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-20T10:31:04,434 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000003000 2024-11-20T10:31:04,435 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/0000000000000006000 2024-11-20T10:31:04,437 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,437 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,438 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:04,440 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec5fd4cececb596c024a28f5c8244da2 2024-11-20T10:31:04,442 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/test2727/ec5fd4cececb596c024a28f5c8244da2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-20T10:31:04,443 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec5fd4cececb596c024a28f5c8244da2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62761711, jitterRate=-0.06477762758731842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:04,444 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec5fd4cececb596c024a28f5c8244da2: Writing region info on filesystem at 1732098664215Initializing all the Stores at 1732098664216 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098664216Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098664216Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098664217 (+1 ms)Obtaining lock to block concurrent updates at 1732098664301 (+84 ms)Preparing flush snapshotting stores in ec5fd4cececb596c024a28f5c8244da2 at 1732098664301Finished memstore snapshotting test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732098664301Flushing stores of test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. at 1732098664301Flushing ec5fd4cececb596c024a28f5c8244da2/a: creating writer at 1732098664301Flushing ec5fd4cececb596c024a28f5c8244da2/a: appending metadata at 1732098664320 (+19 ms)Flushing ec5fd4cececb596c024a28f5c8244da2/a: closing flushed file at 1732098664320Flushing ec5fd4cececb596c024a28f5c8244da2/b: creating writer at 1732098664335 (+15 ms)Flushing ec5fd4cececb596c024a28f5c8244da2/b: appending metadata at 1732098664353 (+18 ms)Flushing ec5fd4cececb596c024a28f5c8244da2/b: closing flushed file at 1732098664353Flushing ec5fd4cececb596c024a28f5c8244da2/c: creating writer at 1732098664366 (+13 ms)Flushing ec5fd4cececb596c024a28f5c8244da2/c: appending metadata at 1732098664391 (+25 ms)Flushing ec5fd4cececb596c024a28f5c8244da2/c: closing flushed file at 1732098664391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42ad19e8: reopening flushed file at 1732098664406 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c92bc4: reopening flushed file at 1732098664415 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f2eece4: reopening flushed file at 1732098664424 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ec5fd4cececb596c024a28f5c8244da2 in 132ms, sequenceid=6000, compaction requested=false; wal=null at 1732098664433 (+9 ms)Cleaning up temporary data from old regions at 1732098664437 (+4 ms)Region opened successfully at 1732098664444 (+7 ms) 2024-11-20T10:31:04,445 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-20T10:31:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec5fd4cececb596c024a28f5c8244da2, disabling compactions & flushes 2024-11-20T10:31:04,446 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:04,446 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:04,446 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. after waiting 0 ms 2024-11-20T10:31:04,446 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:04,447 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732098663618.ec5fd4cececb596c024a28f5c8244da2. 2024-11-20T10:31:04,447 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec5fd4cececb596c024a28f5c8244da2: Waiting for close lock at 1732098664445Disabling compacts and flushes for region at 1732098664445Disabling writes for close at 1732098664446 (+1 ms)Writing region close event to WAL at 1732098664447 (+1 ms)Closed at 1732098664447 2024-11-20T10:31:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741870_1047 (size=93) 2024-11-20T10:31:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741870_1047 (size=93) 2024-11-20T10:31:04,451 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732098663616/wal.1732098664195 not finished, retry = 0 2024-11-20T10:31:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741870_1047 (size=93) 2024-11-20T10:31:04,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:04,557 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732098664195) 2024-11-20T10:31:04,576 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=400 (was 397) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35008 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50266 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57978 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=925 (was 861) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=8386 (was 8530) 2024-11-20T10:31:04,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=400, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=8385 2024-11-20T10:31:04,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:04,611 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:04,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732098664612 2024-11-20T10:31:04,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 2024-11-20T10:31:04,621 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:31:04,622 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aae301647c945d6eee4d268fdd2f2656, NAME => 'testSequentialEditLogSeqNum,,1732098664606.aae301647c945d6eee4d268fdd2f2656.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:04,622 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732098664606.aae301647c945d6eee4d268fdd2f2656.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:04,622 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,622 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,623 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656 doesn't exist for region: aae301647c945d6eee4d268fdd2f2656 on table testSequentialEditLogSeqNum 2024-11-20T10:31:04,624 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: aae301647c945d6eee4d268fdd2f2656 on table testSequentialEditLogSeqNum 2024-11-20T10:31:04,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741875_1052 (size=62) 2024-11-20T10:31:04,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741875_1052 (size=62) 2024-11-20T10:31:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741875_1052 (size=62) 2024-11-20T10:31:04,636 INFO [StoreOpener-aae301647c945d6eee4d268fdd2f2656-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,638 INFO [StoreOpener-aae301647c945d6eee4d268fdd2f2656-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aae301647c945d6eee4d268fdd2f2656 columnFamilyName a 2024-11-20T10:31:04,638 DEBUG [StoreOpener-aae301647c945d6eee4d268fdd2f2656-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:04,639 INFO [StoreOpener-aae301647c945d6eee4d268fdd2f2656-1 {}] regionserver.HStore(327): Store=aae301647c945d6eee4d268fdd2f2656/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:04,639 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,640 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,640 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,642 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aae301647c945d6eee4d268fdd2f2656 2024-11-20T10:31:04,644 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:04,645 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aae301647c945d6eee4d268fdd2f2656; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69921830, jitterRate=0.041916459798812866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:31:04,646 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aae301647c945d6eee4d268fdd2f2656: Writing region info on filesystem at 1732098664622Initializing all the Stores at 1732098664636 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098664636Cleaning up temporary data from old regions at 1732098664641 (+5 ms)Region opened successfully at 1732098664645 (+4 ms) 2024-11-20T10:31:04,656 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aae301647c945d6eee4d268fdd2f2656 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-20T10:31:04,674 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/.tmp/a/8983047881e44c98893a4e2400e36ddf is 81, key is testSequentialEditLogSeqNum/a:x0/1732098664646/Put/seqid=0 2024-11-20T10:31:04,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T10:31:04,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:04,682 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T10:31:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741876_1053 (size=5833) 2024-11-20T10:31:04,682 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T10:31:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741876_1053 (size=5833) 2024-11-20T10:31:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741876_1053 (size=5833) 2024-11-20T10:31:04,683 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T10:31:04,683 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:04,684 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-20T10:31:04,684 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:04,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-20T10:31:04,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:05,084 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/.tmp/a/8983047881e44c98893a4e2400e36ddf 2024-11-20T10:31:05,092 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/.tmp/a/8983047881e44c98893a4e2400e36ddf as hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/a/8983047881e44c98893a4e2400e36ddf 2024-11-20T10:31:05,101 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/a/8983047881e44c98893a4e2400e36ddf, entries=10, sequenceid=13, filesize=5.7 K 2024-11-20T10:31:05,103 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for aae301647c945d6eee4d268fdd2f2656 in 446ms, sequenceid=13, compaction requested=false 2024-11-20T10:31:05,103 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aae301647c945d6eee4d268fdd2f2656: 2024-11-20T10:31:05,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T10:31:05,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T10:31:05,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T10:31:05,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T10:31:05,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T10:31:05,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741874_1051 (size=3017) 2024-11-20T10:31:05,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741874_1051 (size=3017) 2024-11-20T10:31:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741874_1051 (size=3017) 2024-11-20T10:31:05,129 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612, size=2.9 K (3017bytes) 2024-11-20T10:31:05,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 2024-11-20T10:31:05,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 after 0ms 2024-11-20T10:31:05,132 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 took 4ms 2024-11-20T10:31:05,134 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 so closing down 2024-11-20T10:31:05,134 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:05,135 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732098664612.temp 2024-11-20T10:31:05,137 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp 2024-11-20T10:31:05,137 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:05,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741877_1054 (size=2433) 2024-11-20T10:31:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741877_1054 (size=2433) 2024-11-20T10:31:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741877_1054 (size=2433) 2024-11-20T10:31:05,148 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:05,149 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp to hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000020 2024-11-20T10:31:05,149 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-20T10:31:05,149 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612, journal: Splitting hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612, size=2.9 K (3017bytes) at 1732098665129Finishing writing output for hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612 so closing down at 1732098665134 (+5 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp at 1732098665137 (+3 ms)3 split writer threads finished at 1732098665137Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732098665148 (+11 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000003-wal.1732098664612.temp to hdfs://localhost:36923/hbase/data/default/testSequentialEditLogSeqNum/aae301647c945d6eee4d268fdd2f2656/recovered.edits/0000000000000000020 at 1732098665149 (+1 ms)Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732098664604/wal.1732098664612, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732098665149 2024-11-20T10:31:05,161 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=405 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35008 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:57978 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=8381 (was 8385) 2024-11-20T10:31:05,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=8381 2024-11-20T10:31:05,186 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:05,188 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:05,220 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:05,224 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63825339, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-63825339, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:05,237 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63825339/hregion-63825339.1732098665224, exclude list is [], retry=0 2024-11-20T10:31:05,239 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:05,240 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:05,240 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:05,242 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63825339/hregion-63825339.1732098665224 2024-11-20T10:31:05,242 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:31:05,243 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 16ffc1e1751c24841f59b1ad02f4e285, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:05,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741879_1056 (size=70) 2024-11-20T10:31:05,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741879_1056 (size=70) 2024-11-20T10:31:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741879_1056 (size=70) 2024-11-20T10:31:05,253 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:05,256 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,258 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName a 2024-11-20T10:31:05,258 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,259 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,259 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,260 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName b 2024-11-20T10:31:05,260 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,261 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,261 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,262 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName c 2024-11-20T10:31:05,263 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,263 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,263 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,266 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,266 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,266 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:05,267 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,270 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:05,271 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 16ffc1e1751c24841f59b1ad02f4e285; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67157785, jitterRate=7.289797067642212E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:05,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 16ffc1e1751c24841f59b1ad02f4e285: Writing region info on filesystem at 1732098665253Initializing all the Stores at 1732098665254 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665254Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665256 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665256Cleaning up temporary data from old regions at 1732098665266 (+10 ms)Region opened successfully at 1732098665271 (+5 ms) 2024-11-20T10:31:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 16ffc1e1751c24841f59b1ad02f4e285, disabling compactions & flushes 2024-11-20T10:31:05,272 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. after waiting 0 ms 2024-11-20T10:31:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:05,272 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:05,272 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 16ffc1e1751c24841f59b1ad02f4e285: Waiting for close lock at 1732098665271Disabling compacts and flushes for region at 1732098665271Disabling writes for close at 1732098665272 (+1 ms)Writing region close event to WAL at 1732098665272Closed at 1732098665272 2024-11-20T10:31:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741878_1055 (size=93) 2024-11-20T10:31:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741878_1055 (size=93) 2024-11-20T10:31:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741878_1055 (size=93) 2024-11-20T10:31:05,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:05,278 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63825339:(num 1732098665224) 2024-11-20T10:31:05,278 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:05,280 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:05,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, exclude list is [], retry=0 2024-11-20T10:31:05,296 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:05,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:05,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:05,299 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 2024-11-20T10:31:05,300 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:05,300 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 16ffc1e1751c24841f59b1ad02f4e285, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:05,300 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:05,300 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,301 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,302 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,303 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName a 2024-11-20T10:31:05,303 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,304 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,304 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,305 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName b 2024-11-20T10:31:05,305 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,306 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,306 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,307 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName c 2024-11-20T10:31:05,307 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:05,308 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:05,308 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,309 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,310 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,311 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,311 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,312 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:05,313 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:05,314 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 16ffc1e1751c24841f59b1ad02f4e285; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75109126, jitterRate=0.11921319365501404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:05,315 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 16ffc1e1751c24841f59b1ad02f4e285: Writing region info on filesystem at 1732098665301Initializing all the Stores at 1732098665302 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665302Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665302Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098665302Cleaning up temporary data from old regions at 1732098665311 (+9 ms)Region opened successfully at 1732098665315 (+4 ms) 2024-11-20T10:31:05,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732098665318/Put/seqid=0 2024-11-20T10:31:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741881_1058 (size=4826) 2024-11-20T10:31:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741881_1058 (size=4826) 2024-11-20T10:31:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741881_1058 (size=4826) 2024-11-20T10:31:05,327 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36923/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 16ffc1e1751c24841f59b1ad02f4e285/a 2024-11-20T10:31:05,334 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-20T10:31:05,334 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T10:31:05,334 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 16ffc1e1751c24841f59b1ad02f4e285: 2024-11-20T10:31:05,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/97a30c0613484c608f8fcebc062f9286_SeqId_3_ 2024-11-20T10:31:05,337 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36923/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 16ffc1e1751c24841f59b1ad02f4e285/a as hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/97a30c0613484c608f8fcebc062f9286_SeqId_3_ - updating store file list. 2024-11-20T10:31:05,343 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 97a30c0613484c608f8fcebc062f9286_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:05,343 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/97a30c0613484c608f8fcebc062f9286_SeqId_3_ into 16ffc1e1751c24841f59b1ad02f4e285/a 2024-11-20T10:31:05,343 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36923/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 16ffc1e1751c24841f59b1ad02f4e285/a (new location: hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/97a30c0613484c608f8fcebc062f9286_SeqId_3_) 2024-11-20T10:31:05,379 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, size=0 (0bytes) 2024-11-20T10:31:05,379 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 might be still open, length is 0 2024-11-20T10:31:05,379 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 2024-11-20T10:31:05,380 WARN [IPC Server handler 4 on default port 36923 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-20T10:31:05,380 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 after 1ms 2024-11-20T10:31:05,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35200 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35200 dst: /127.0.0.1:39533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39533 remote=/127.0.0.1:35200]. Total timeout mills is 60000, 59930 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:05,420 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:58062 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46093:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58062 dst: /127.0.0.1:46093 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:05,420 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50478 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50478 dst: /127.0.0.1:38347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:05,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741880_1059 (size=504) 2024-11-20T10:31:05,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741880_1059 (size=504) 2024-11-20T10:31:09,381 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 after 4001ms 2024-11-20T10:31:09,384 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 took 4005ms 2024-11-20T10:31:09,387 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280; continuing. 2024-11-20T10:31:09,387 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 so closing down 2024-11-20T10:31:09,387 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:09,389 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732098665280.temp 2024-11-20T10:31:09,391 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp 2024-11-20T10:31:09,391 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:09,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741882_1060 (size=238) 2024-11-20T10:31:09,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741882_1060 (size=238) 2024-11-20T10:31:09,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741882_1060 (size=238) 2024-11-20T10:31:09,401 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:09,403 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp to hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 2024-11-20T10:31:09,403 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T10:31:09,403 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, journal: Splitting hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, size=0 (0bytes) at 1732098665379Finishing writing output for hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 so closing down at 1732098669387 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp at 1732098669391 (+4 ms)3 split writer threads finished at 1732098669391Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732098669401 (+10 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005-wal.1732098665280.temp to hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 at 1732098669403 (+2 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280, size=0, length=0, corrupted=false, cancelled=false at 1732098669403 2024-11-20T10:31:09,405 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098665280 2024-11-20T10:31:09,406 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 2024-11-20T10:31:09,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:09,408 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:09,419 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098669408, exclude list is [], retry=0 2024-11-20T10:31:09,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:09,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:09,423 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:09,424 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098669408 2024-11-20T10:31:09,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:31:09,425 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 16ffc1e1751c24841f59b1ad02f4e285, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:09,425 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:09,425 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,425 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,426 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,427 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName a 2024-11-20T10:31:09,428 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,433 DEBUG [StoreFileOpener-16ffc1e1751c24841f59b1ad02f4e285-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 97a30c0613484c608f8fcebc062f9286_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:09,433 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/97a30c0613484c608f8fcebc062f9286_SeqId_3_ 2024-11-20T10:31:09,434 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:09,434 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,435 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName b 2024-11-20T10:31:09,435 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,435 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:09,435 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,436 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16ffc1e1751c24841f59b1ad02f4e285 columnFamilyName c 2024-11-20T10:31:09,436 DEBUG [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,437 INFO [StoreOpener-16ffc1e1751c24841f59b1ad02f4e285-1 {}] regionserver.HStore(327): Store=16ffc1e1751c24841f59b1ad02f4e285/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,438 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,439 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,440 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 2024-11-20T10:31:09,443 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 2024-11-20T10:31:09,443 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 16ffc1e1751c24841f59b1ad02f4e285 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-20T10:31:09,458 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/.tmp/a/6cf8d58cbba844b196cf6d400b79ec0e is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732098665347/Put/seqid=0 2024-11-20T10:31:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741884_1062 (size=5149) 2024-11-20T10:31:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741884_1062 (size=5149) 2024-11-20T10:31:09,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741884_1062 (size=5149) 2024-11-20T10:31:09,465 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/.tmp/a/6cf8d58cbba844b196cf6d400b79ec0e 2024-11-20T10:31:09,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/.tmp/a/6cf8d58cbba844b196cf6d400b79ec0e as hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/6cf8d58cbba844b196cf6d400b79ec0e 2024-11-20T10:31:09,476 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/a/6cf8d58cbba844b196cf6d400b79ec0e, entries=1, sequenceid=5, filesize=5.0 K 2024-11-20T10:31:09,476 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 16ffc1e1751c24841f59b1ad02f4e285 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-20T10:31:09,477 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/0000000000000000005 2024-11-20T10:31:09,478 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,478 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,478 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:09,480 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 16ffc1e1751c24841f59b1ad02f4e285 2024-11-20T10:31:09,482 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/16ffc1e1751c24841f59b1ad02f4e285/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-20T10:31:09,483 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 16ffc1e1751c24841f59b1ad02f4e285; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63253439, jitterRate=-0.0574503093957901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:09,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 16ffc1e1751c24841f59b1ad02f4e285: Writing region info on filesystem at 1732098669425Initializing all the Stores at 1732098669426 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098669426Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098669426Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098669426Obtaining lock to block concurrent updates at 1732098669443 (+17 ms)Preparing flush snapshotting stores in 16ffc1e1751c24841f59b1ad02f4e285 at 1732098669443Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732098669443Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. at 1732098669443Flushing 16ffc1e1751c24841f59b1ad02f4e285/a: creating writer at 1732098669443Flushing 16ffc1e1751c24841f59b1ad02f4e285/a: appending metadata at 1732098669457 (+14 ms)Flushing 16ffc1e1751c24841f59b1ad02f4e285/a: closing flushed file at 1732098669457Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3828ae8d: reopening flushed file at 1732098669469 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 16ffc1e1751c24841f59b1ad02f4e285 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1732098669476 (+7 ms)Cleaning up temporary data from old regions at 1732098669478 (+2 ms)Region opened successfully at 1732098669483 (+5 ms) 2024-11-20T10:31:09,487 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 16ffc1e1751c24841f59b1ad02f4e285, disabling compactions & flushes 2024-11-20T10:31:09,487 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:09,487 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:09,487 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. after waiting 0 ms 2024-11-20T10:31:09,487 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:09,488 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732098665187.16ffc1e1751c24841f59b1ad02f4e285. 2024-11-20T10:31:09,488 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 16ffc1e1751c24841f59b1ad02f4e285: Waiting for close lock at 1732098669487Disabling compacts and flushes for region at 1732098669487Disabling writes for close at 1732098669487Writing region close event to WAL at 1732098669488 (+1 ms)Closed at 1732098669488 2024-11-20T10:31:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741883_1061 (size=93) 2024-11-20T10:31:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741883_1061 (size=93) 2024-11-20T10:31:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741883_1061 (size=93) 2024-11-20T10:31:09,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:09,493 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732098669408) 2024-11-20T10:31:09,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 405) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_232988075_22 at /127.0.0.1:50494 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36923 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_232988075_22 at /127.0.0.1:35234 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:36923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=8378 (was 8381) 2024-11-20T10:31:09,515 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=8375 2024-11-20T10:31:09,529 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:09,534 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T10:31:09,538 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8c533afd331c,38325,1732098654332 2024-11-20T10:31:09,540 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ed892f6 2024-11-20T10:31:09,541 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T10:31:09,543 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T10:31:09,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T10:31:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T10:31:09,554 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T10:31:09,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-20T10:31:09,556 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,558 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T10:31:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:31:09,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741885_1063 (size=694) 2024-11-20T10:31:09,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741885_1063 (size=694) 2024-11-20T10:31:09,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741885_1063 (size=694) 2024-11-20T10:31:09,571 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87 2024-11-20T10:31:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741886_1064 (size=77) 2024-11-20T10:31:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741886_1064 (size=77) 2024-11-20T10:31:09,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741886_1064 (size=77) 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:09,581 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,581 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,581 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098669581Disabling compacts and flushes for region at 1732098669581Disabling writes for close at 1732098669581Writing region close event to WAL at 1732098669581Closed at 1732098669581 2024-11-20T10:31:09,583 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T10:31:09,589 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732098669583"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732098669583"}]},"ts":"1732098669583"} 2024-11-20T10:31:09,593 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T10:31:09,595 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T10:31:09,598 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098669595"}]},"ts":"1732098669595"} 2024-11-20T10:31:09,602 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-20T10:31:09,603 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(204): Hosts are {8c533afd331c=0} racks are {/default-rack=0} 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T10:31:09,604 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T10:31:09,604 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T10:31:09,604 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T10:31:09,604 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T10:31:09,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN}] 2024-11-20T10:31:09,607 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN 2024-11-20T10:31:09,609 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN; state=OFFLINE, location=8c533afd331c,41755,1732098655250; forceNewPlan=false, retain=false 2024-11-20T10:31:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:31:09,761 INFO [8c533afd331c:38325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T10:31:09,762 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPENING, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:31:09,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN because future has completed 2024-11-20T10:31:09,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250}] 2024-11-20T10:31:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:31:09,926 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,926 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:09,927 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,927 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:09,927 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,927 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,929 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,930 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:09,930 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,931 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:09,931 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,933 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:09,933 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:09,933 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:09,934 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,935 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,935 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,936 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,936 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,937 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T10:31:09,939 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,941 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:09,942 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 73ab4c4b40d6b3373ca57987a514eca4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71524894, jitterRate=0.06580397486686707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T10:31:09,942 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:09,943 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 73ab4c4b40d6b3373ca57987a514eca4: Running coprocessor pre-open hook at 1732098669927Writing region info on filesystem at 1732098669927Initializing all the Stores at 1732098669928 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098669928Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098669928Cleaning up temporary data from old regions at 1732098669936 (+8 ms)Running coprocessor post-open hooks at 1732098669942 (+6 ms)Region opened successfully at 1732098669943 (+1 ms) 2024-11-20T10:31:09,944 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., pid=6, masterSystemTime=1732098669919 2024-11-20T10:31:09,947 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,948 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:09,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPEN, openSeqNum=2, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:31:09,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 because future has completed 2024-11-20T10:31:09,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T10:31:09,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 in 188 msec 2024-11-20T10:31:09,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T10:31:09,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN in 353 msec 2024-11-20T10:31:09,963 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T10:31:09,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732098669963"}]},"ts":"1732098669963"} 2024-11-20T10:31:09,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-20T10:31:09,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T10:31:09,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 420 msec 2024-11-20T10:31:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T10:31:10,192 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-20T10:31:10,192 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-20T10:31:10,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T10:31:10,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-20T10:31:10,204 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T10:31:10,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-20T10:31:10,218 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=2] 2024-11-20T10:31:10,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=73ab4c4b40d6b3373ca57987a514eca4, source=8c533afd331c,41755,1732098655250, destination=8c533afd331c,40989,1732098655109, warming up region on 8c533afd331c,40989,1732098655109 2024-11-20T10:31:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:31:10,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=73ab4c4b40d6b3373ca57987a514eca4, source=8c533afd331c,41755,1732098655250, destination=8c533afd331c,40989,1732098655109, running balancer 2024-11-20T10:31:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55389, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:31:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE 2024-11-20T10:31:10,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE 2024-11-20T10:31:10,242 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=CLOSING, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:31:10,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(7855): Warmup {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:10,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE because future has completed 2024-11-20T10:31:10,245 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,245 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:31:10,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250}] 2024-11-20T10:31:10,246 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:10,246 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:10,247 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:10,247 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,248 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:10,249 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:10,249 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:10,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098670249Disabling compacts and flushes for region at 1732098670249Disabling writes for close at 1732098670250 (+1 ms)Writing region close event to WAL at 1732098670250Closed at 1732098670250 2024-11-20T10:31:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T10:31:10,405 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,406 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:31:10,407 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:10,407 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,407 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,407 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:10,407 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,408 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 73ab4c4b40d6b3373ca57987a514eca4 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-20T10:31:10,429 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/28dbda6c96e241e69a90718b3d57f4ba is 35, key is r1/cf1:q/1732098670220/Put/seqid=0 2024-11-20T10:31:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741887_1065 (size=4783) 2024-11-20T10:31:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741887_1065 (size=4783) 2024-11-20T10:31:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741887_1065 (size=4783) 2024-11-20T10:31:10,437 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/28dbda6c96e241e69a90718b3d57f4ba 2024-11-20T10:31:10,445 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/28dbda6c96e241e69a90718b3d57f4ba as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba 2024-11-20T10:31:10,451 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba, entries=1, sequenceid=5, filesize=4.7 K 2024-11-20T10:31:10,453 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 73ab4c4b40d6b3373ca57987a514eca4 in 44ms, sequenceid=5, compaction requested=false 2024-11-20T10:31:10,453 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-20T10:31:10,458 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T10:31:10,460 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,460 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098670407Running coprocessor pre-close hooks at 1732098670407Disabling compacts and flushes for region at 1732098670407Disabling writes for close at 1732098670407Obtaining lock to block concurrent updates at 1732098670408 (+1 ms)Preparing flush snapshotting stores in 73ab4c4b40d6b3373ca57987a514eca4 at 1732098670408Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732098670408Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. at 1732098670410 (+2 ms)Flushing 73ab4c4b40d6b3373ca57987a514eca4/cf1: creating writer at 1732098670410Flushing 73ab4c4b40d6b3373ca57987a514eca4/cf1: appending metadata at 1732098670428 (+18 ms)Flushing 73ab4c4b40d6b3373ca57987a514eca4/cf1: closing flushed file at 1732098670429 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10e19f0: reopening flushed file at 1732098670444 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 73ab4c4b40d6b3373ca57987a514eca4 in 44ms, sequenceid=5, compaction requested=false at 1732098670453 (+9 ms)Writing region close event to WAL at 1732098670454 (+1 ms)Running coprocessor post-close hooks at 1732098670458 (+4 ms)Closed at 1732098670460 (+2 ms) 2024-11-20T10:31:10,461 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 73ab4c4b40d6b3373ca57987a514eca4 move to 8c533afd331c,40989,1732098655109 record at close sequenceid=5 2024-11-20T10:31:10,463 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,464 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=CLOSED 2024-11-20T10:31:10,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 because future has completed 2024-11-20T10:31:10,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T10:31:10,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 in 223 msec 2024-11-20T10:31:10,473 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE; state=CLOSED, location=8c533afd331c,40989,1732098655109; forceNewPlan=false, retain=false 2024-11-20T10:31:10,623 INFO [8c533afd331c:38325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T10:31:10,624 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPENING, regionLocation=8c533afd331c,40989,1732098655109 2024-11-20T10:31:10,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE because future has completed 2024-11-20T10:31:10,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109}] 2024-11-20T10:31:10,791 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,792 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:10,792 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,793 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:10,793 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,793 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,795 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,796 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:10,796 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:10,806 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba 2024-11-20T10:31:10,806 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:10,806 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,807 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:10,807 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:10,807 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:10,808 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,808 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,810 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,811 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,811 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,811 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T10:31:10,813 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,814 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 73ab4c4b40d6b3373ca57987a514eca4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67558048, jitterRate=0.006693363189697266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T10:31:10,814 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:10,815 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 73ab4c4b40d6b3373ca57987a514eca4: Running coprocessor pre-open hook at 1732098670793Writing region info on filesystem at 1732098670793Initializing all the Stores at 1732098670794 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098670794Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098670795 (+1 ms)Cleaning up temporary data from old regions at 1732098670811 (+16 ms)Running coprocessor post-open hooks at 1732098670814 (+3 ms)Region opened successfully at 1732098670815 (+1 ms) 2024-11-20T10:31:10,816 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., pid=9, masterSystemTime=1732098670785 2024-11-20T10:31:10,818 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,818 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:10,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPEN, openSeqNum=9, regionLocation=8c533afd331c,40989,1732098655109 2024-11-20T10:31:10,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 because future has completed 2024-11-20T10:31:10,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T10:31:10,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 in 193 msec 2024-11-20T10:31:10,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE in 587 msec 2024-11-20T10:31:10,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:31:10,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:31:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:38000 deadline: 1732098730851, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=40989 startCode=1732098655109. As of locationSeqNum=5. 2024-11-20T10:31:10,873 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=40989 startCode=1732098655109. As of locationSeqNum=5. 2024-11-20T10:31:10,874 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=40989 startCode=1732098655109. As of locationSeqNum=5. 2024-11-20T10:31:10,874 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=40989 startCode=1732098655109. As of locationSeqNum=5. 2024-11-20T10:31:10,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:31:10,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:31:10,992 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 73ab4c4b40d6b3373ca57987a514eca4 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-20T10:31:11,008 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/5538089fe93343a0b258dc55823c0ef6 is 29, key is r1/cf1:/1732098670982/DeleteFamily/seqid=0 2024-11-20T10:31:11,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741888_1066 (size=4906) 2024-11-20T10:31:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741888_1066 (size=4906) 2024-11-20T10:31:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741888_1066 (size=4906) 2024-11-20T10:31:11,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,021 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,035 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf2/92d6460ea2f34511b11022517d9d7243 is 29, key is r1/cf2:/1732098670982/DeleteFamily/seqid=0 2024-11-20T10:31:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741889_1067 (size=4906) 2024-11-20T10:31:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741889_1067 (size=4906) 2024-11-20T10:31:11,043 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf2/92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741889_1067 (size=4906) 2024-11-20T10:31:11,049 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/5538089fe93343a0b258dc55823c0ef6 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,057 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,057 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T10:31:11,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf2/92d6460ea2f34511b11022517d9d7243 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,064 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T10:31:11,065 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 73ab4c4b40d6b3373ca57987a514eca4 in 73ms, sequenceid=12, compaction requested=false 2024-11-20T10:31:11,065 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 73ab4c4b40d6b3373ca57987a514eca4: 2024-11-20T10:31:11,068 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T10:31:11,069 DEBUG [Time-limited test {}] regionserver.HStore(1541): 73ab4c4b40d6b3373ca57987a514eca4/cf1 is initiating major compaction (all files) 2024-11-20T10:31:11,069 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:31:11,069 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:31:11,069 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 73ab4c4b40d6b3373ca57987a514eca4/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,070 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba, hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6] into tmpdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp, totalSize=9.5 K 2024-11-20T10:31:11,071 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 28dbda6c96e241e69a90718b3d57f4ba, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732098670220 2024-11-20T10:31:11,072 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5538089fe93343a0b258dc55823c0ef6, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T10:31:11,084 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 73ab4c4b40d6b3373ca57987a514eca4#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:31:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741890_1068 (size=4626) 2024-11-20T10:31:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741890_1068 (size=4626) 2024-11-20T10:31:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741890_1068 (size=4626) 2024-11-20T10:31:11,098 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf1/f243daf9d2264427a6f57c43eb5ffa24 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/f243daf9d2264427a6f57c43eb5ffa24 2024-11-20T10:31:11,111 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 73ab4c4b40d6b3373ca57987a514eca4/cf1 of 73ab4c4b40d6b3373ca57987a514eca4 into f243daf9d2264427a6f57c43eb5ffa24(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:31:11,111 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 73ab4c4b40d6b3373ca57987a514eca4: 2024-11-20T10:31:11,111 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-20T10:31:11,111 DEBUG [Time-limited test {}] regionserver.HStore(1541): 73ab4c4b40d6b3373ca57987a514eca4/cf2 is initiating major compaction (all files) 2024-11-20T10:31:11,111 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T10:31:11,111 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T10:31:11,112 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 73ab4c4b40d6b3373ca57987a514eca4/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,112 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243] into tmpdir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp, totalSize=4.8 K 2024-11-20T10:31:11,112 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 92d6460ea2f34511b11022517d9d7243, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T10:31:11,118 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 73ab4c4b40d6b3373ca57987a514eca4#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T10:31:11,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741891_1069 (size=4592) 2024-11-20T10:31:11,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741891_1069 (size=4592) 2024-11-20T10:31:11,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741891_1069 (size=4592) 2024-11-20T10:31:11,131 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/.tmp/cf2/861b8e841b484ae1835cc5e0d2cfd2f1 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/861b8e841b484ae1835cc5e0d2cfd2f1 2024-11-20T10:31:11,138 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 73ab4c4b40d6b3373ca57987a514eca4/cf2 of 73ab4c4b40d6b3373ca57987a514eca4 into 861b8e841b484ae1835cc5e0d2cfd2f1(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:31:11,138 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 73ab4c4b40d6b3373ca57987a514eca4: 2024-11-20T10:31:11,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=73ab4c4b40d6b3373ca57987a514eca4, source=8c533afd331c,40989,1732098655109, destination=8c533afd331c,41755,1732098655250, warming up region on 8c533afd331c,41755,1732098655250 2024-11-20T10:31:11,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=73ab4c4b40d6b3373ca57987a514eca4, source=8c533afd331c,40989,1732098655109, destination=8c533afd331c,41755,1732098655250, running balancer 2024-11-20T10:31:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE 2024-11-20T10:31:11,144 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE 2024-11-20T10:31:11,145 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=CLOSING, regionLocation=8c533afd331c,40989,1732098655109 2024-11-20T10:31:11,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(7855): Warmup {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:11,147 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE because future has completed 2024-11-20T10:31:11,148 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:11,148 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:11,148 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T10:31:11,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109}] 2024-11-20T10:31:11,155 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba 2024-11-20T10:31:11,160 INFO [StoreFileOpener-73ab4c4b40d6b3373ca57987a514eca4-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,160 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,166 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/f243daf9d2264427a6f57c43eb5ffa24 2024-11-20T10:31:11,166 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:11,166 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,167 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:11,167 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:11,173 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/861b8e841b484ae1835cc5e0d2cfd2f1 2024-11-20T10:31:11,178 INFO [StoreFileOpener-73ab4c4b40d6b3373ca57987a514eca4-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,178 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,178 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:11,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41755 {}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098671178Disabling compacts and flushes for region at 1732098671178Disabling writes for close at 1732098671178Writing region close event to WAL at 1732098671179 (+1 ms)Closed at 1732098671179 2024-11-20T10:31:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-20T10:31:11,302 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,303 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T10:31:11,303 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:11,303 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,303 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,303 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:11,303 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,303 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba, hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6] to archive 2024-11-20T10:31:11,306 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:31:11,310 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/28dbda6c96e241e69a90718b3d57f4ba 2024-11-20T10:31:11,312 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6 to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/5538089fe93343a0b258dc55823c0ef6 2024-11-20T10:31:11,325 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243] to archive 2024-11-20T10:31:11,326 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:31:11,327 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243 to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/92d6460ea2f34511b11022517d9d7243 2024-11-20T10:31:11,333 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-20T10:31:11,334 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,334 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098671303Running coprocessor pre-close hooks at 1732098671303Disabling compacts and flushes for region at 1732098671303Disabling writes for close at 1732098671303Writing region close event to WAL at 1732098671329 (+26 ms)Running coprocessor post-close hooks at 1732098671334 (+5 ms)Closed at 1732098671334 2024-11-20T10:31:11,334 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 73ab4c4b40d6b3373ca57987a514eca4 move to 8c533afd331c,41755,1732098655250 record at close sequenceid=12 2024-11-20T10:31:11,337 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,338 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=CLOSED 2024-11-20T10:31:11,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 because future has completed 2024-11-20T10:31:11,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T10:31:11,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 in 194 msec 2024-11-20T10:31:11,345 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE; state=CLOSED, location=8c533afd331c,41755,1732098655250; forceNewPlan=false, retain=false 2024-11-20T10:31:11,496 INFO [8c533afd331c:38325 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T10:31:11,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPENING, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:31:11,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE because future has completed 2024-11-20T10:31:11,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250}] 2024-11-20T10:31:11,665 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,665 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:11,665 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,665 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:11,665 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,666 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,667 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,668 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:11,669 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:11,677 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/f243daf9d2264427a6f57c43eb5ffa24 2024-11-20T10:31:11,677 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:11,677 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,678 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:11,678 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:11,685 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/861b8e841b484ae1835cc5e0d2cfd2f1 2024-11-20T10:31:11,685 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:11,685 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,686 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,687 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,688 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,688 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,689 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T10:31:11,690 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,691 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 73ab4c4b40d6b3373ca57987a514eca4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66387827, jitterRate=-0.010744288563728333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T10:31:11,691 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,692 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 73ab4c4b40d6b3373ca57987a514eca4: Running coprocessor pre-open hook at 1732098671666Writing region info on filesystem at 1732098671666Initializing all the Stores at 1732098671667 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098671667Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098671667Cleaning up temporary data from old regions at 1732098671688 (+21 ms)Running coprocessor post-open hooks at 1732098671691 (+3 ms)Region opened successfully at 1732098671692 (+1 ms) 2024-11-20T10:31:11,693 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., pid=12, masterSystemTime=1732098671654 2024-11-20T10:31:11,696 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,696 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPEN, openSeqNum=18, regionLocation=8c533afd331c,41755,1732098655250 2024-11-20T10:31:11,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 because future has completed 2024-11-20T10:31:11,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-20T10:31:11,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,41755,1732098655250 in 200 msec 2024-11-20T10:31:11,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, REOPEN/MOVE in 561 msec 2024-11-20T10:31:11,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:31:11,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:31:11,749 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 8c533afd331c,41755,1732098655250: testing ***** 2024-11-20T10:31:11,749 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T10:31:11,752 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T10:31:11,753 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T10:31:11,757 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T10:31:11,758 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T10:31:11,767 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 449627024 }, "NonHeapMemoryUsage": { "committed": 170262528, "init": 7667712, "max": -1, "used": 167665896 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "8c533afd331c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2077, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 5, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 1, "QueueCallTime_median": 2, "QueueCallTime_75th_percentile": 3, "QueueCallTime_90th_percentile": 4, "QueueCallTime_95th_percentile": 4, "QueueCallTime_98th_percentile": 4, "QueueCallTime_99th_percentile": 4, "QueueCallTime_99.9th_percentile": 4, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 14, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 12, "TotalCallTime_95th_percentile": 13, "TotalCallTime_98th_percentile": 13, "TotalCallTime_99th_percentile": 13, "TotalCallTime_99.9th_percentile": 13, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "8c533afd331c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63828", "tag.serverName": "8c533afd331c,40989,1732098655109", "tag.clusterId": "43f2778b-6f98-41d1-8c84-f7e0d018996c", "tag.Context": "regionserver", "tag.Hostname": "8c533afd331c", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732098655109, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T10:31:11,771 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38325 {}] master.MasterRpcServices(700): 8c533afd331c,41755,1732098655250 reported a fatal error: ***** ABORTING region server 8c533afd331c,41755,1732098655250: testing ***** 2024-11-20T10:31:11,773 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8c533afd331c,41755,1732098655250' ***** 2024-11-20T10:31:11,773 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T10:31:11,774 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(3091): Received CLOSE for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(956): aborting server 8c533afd331c,41755,1732098655250 2024-11-20T10:31:11,774 INFO [RS:2;8c533afd331c:41755 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:31:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40989 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:52374 deadline: 1732098731774, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=41755 startCode=1732098655250. As of locationSeqNum=12. 2024-11-20T10:31:11,775 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:11,775 INFO [RS:2;8c533afd331c:41755 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;8c533afd331c:41755. 2024-11-20T10:31:11,775 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,775 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,775 DEBUG [RS:2;8c533afd331c:41755 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:31:11,775 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:11,775 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,775 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=41755 startCode=1732098655250. As of locationSeqNum=12. 2024-11-20T10:31:11,775 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=41755 startCode=1732098655250. As of locationSeqNum=12. 2024-11-20T10:31:11,775 DEBUG [RS:2;8c533afd331c:41755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:11,775 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8c533afd331c port=41755 startCode=1732098655250. As of locationSeqNum=12. 2024-11-20T10:31:11,776 INFO [RS:2;8c533afd331c:41755 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T10:31:11,776 INFO [RS:2;8c533afd331c:41755 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T10:31:11,776 INFO [RS:2;8c533afd331c:41755 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T10:31:11,776 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T10:31:11,776 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T10:31:11,777 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1325): Online Regions={73ab4c4b40d6b3373ca57987a514eca4=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T10:31:11,777 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T10:31:11,777 DEBUG [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:11,777 ERROR [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-20T10:31:11,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T10:31:11,777 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T10:31:11,777 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732098671776Running coprocessor pre-close hooks at 1732098671776Disabling compacts and flushes for region at 1732098671776Disabling writes for close at 1732098671777 (+1 ms)Writing region close event to WAL at 1732098671777Running coprocessor post-close hooks at 1732098671777Closed at 1732098671777 2024-11-20T10:31:11,777 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098671774Running coprocessor pre-close hooks at 1732098671775 (+1 ms)Disabling compacts and flushes for region at 1732098671775Disabling writes for close at 1732098671775Writing region close event to WAL at 1732098671777 (+2 ms)Running coprocessor post-close hooks at 1732098671777Closed at 1732098671777 2024-11-20T10:31:11,778 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T10:31:11,778 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:11,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting 2024-11-20T10:31:11,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting 2024-11-20T10:31:11,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 from cache 2024-11-20T10:31:11,863 INFO [regionserver/8c533afd331c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:11,889 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:11,890 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1] 2024-11-20T10:31:11,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:11,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting 2024-11-20T10:31:11,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8c533afd331c,41755,1732098655250 aborting 2024-11-20T10:31:11,892 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 from cache 2024-11-20T10:31:11,977 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(976): stopping server 8c533afd331c,41755,1732098655250; all regions closed. 2024-11-20T10:31:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741836_1012 (size=6845) 2024-11-20T10:31:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741836_1012 (size=6845) 2024-11-20T10:31:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741836_1012 (size=6845) 2024-11-20T10:31:11,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741833_1009 (size=2142) 2024-11-20T10:31:11,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741833_1009 (size=2142) 2024-11-20T10:31:11,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741833_1009 (size=2142) 2024-11-20T10:31:11,985 DEBUG [RS:2;8c533afd331c:41755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:11,985 INFO [RS:2;8c533afd331c:41755 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:11,985 INFO [RS:2;8c533afd331c:41755 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:31:11,985 INFO [RS:2;8c533afd331c:41755 {}] hbase.ChoreService(370): Chore service for: regionserver/8c533afd331c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T10:31:11,985 INFO [RS:2;8c533afd331c:41755 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:31:11,985 INFO [regionserver/8c533afd331c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:31:11,986 INFO [RS:2;8c533afd331c:41755 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41755 2024-11-20T10:31:12,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8c533afd331c,41755,1732098655250 2024-11-20T10:31:12,056 INFO [RS:2;8c533afd331c:41755 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:31:12,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T10:31:12,057 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$369/0x00007f98988e3ab0@393b4937 rejected from java.util.concurrent.ThreadPoolExecutor@7c39c074[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 12] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-20T10:31:12,057 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8c533afd331c,41755,1732098655250] 2024-11-20T10:31:12,077 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8c533afd331c,41755,1732098655250 already deleted, retry=false 2024-11-20T10:31:12,078 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 8c533afd331c,41755,1732098655250 on 8c533afd331c,38325,1732098654332 2024-11-20T10:31:12,083 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 8c533afd331c,41755,1732098655250, splitWal=true, meta=true 2024-11-20T10:31:12,086 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 8c533afd331c,41755,1732098655250 (carryingMeta=true) 8c533afd331c,41755,1732098655250/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@35a55257[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T10:31:12,086 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 8c533afd331c,41755,1732098655250, splitWal=true, meta=true 2024-11-20T10:31:12,087 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 8c533afd331c,41755,1732098655250, splitWal=true, meta=true, isMeta: true 2024-11-20T10:31:12,089 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting 2024-11-20T10:31:12,090 INFO [PEWorker-1 {}] master.SplitWALManager(105): 8c533afd331c,41755,1732098655250 WAL count=1, meta=true 2024-11-20T10:31:12,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta}] 2024-11-20T10:31:12,098 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=8c533afd331c,38107,1732098655202 2024-11-20T10:31:12,099 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:12,100 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1] 2024-11-20T10:31:12,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, worker=8c533afd331c,38107,1732098655202}] 2024-11-20T10:31:12,102 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8c533afd331c:41755 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8c533afd331c/172.17.0.2:41755 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:12,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1, error=java.net.ConnectException: Call to address=8c533afd331c:41755 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8c533afd331c/172.17.0.2:41755 2024-11-20T10:31:12,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-20T10:31:12,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 from cache 2024-11-20T10:31:12,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 8c533afd331c:41755 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8c533afd331c/172.17.0.2:41755 2024-11-20T10:31:12,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:12,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41755-0x10157cbf93d0003, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:12,168 INFO [RS:2;8c533afd331c:41755 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:31:12,168 INFO [RS:2;8c533afd331c:41755 {}] regionserver.HRegionServer(1031): Exiting; stopping=8c533afd331c,41755,1732098655250; zookeeper connection closed. 2024-11-20T10:31:12,168 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16c935c8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16c935c8 2024-11-20T10:31:12,264 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T10:31:12,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54587, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T10:31:12,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38107 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-20T10:31:12,291 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, size=6.7 K (6845bytes) 2024-11-20T10:31:12,292 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta 2024-11-20T10:31:12,292 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta after 0ms 2024-11-20T10:31:12,295 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta took 4ms 2024-11-20T10:31:12,301 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T10:31:12,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta so closing down 2024-11-20T10:31:12,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:12,303 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp 2024-11-20T10:31:12,308 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp 2024-11-20T10:31:12,308 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:12,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741892_1070 (size=6604) 2024-11-20T10:31:12,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741892_1070 (size=6604) 2024-11-20T10:31:12,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741892_1070 (size=6604) 2024-11-20T10:31:12,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:12,326 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T10:31:12,326 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 31 ms; skipped=1; WAL=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-20T10:31:12,326 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, journal: Splitting hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, size=6.7 K (6845bytes) at 1732098672291Finishing writing output for hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta so closing down at 1732098672302 (+11 ms)Creating recovered edits writer path=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp at 1732098672308 (+6 ms)3 split writer threads finished at 1732098672308Closed recovered edits writer path=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732098672324 (+16 ms)Rename recovered edits hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta.temp to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732098672326 (+2 ms)Processed 16 edits across 1 Regions in 31 ms; skipped=1; WAL=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732098672326 2024-11-20T10:31:12,326 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta 2024-11-20T10:31:12,328 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-20T10:31:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-20T10:31:12,334 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs 2024-11-20T10:31:12,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T10:31:12,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, worker=8c533afd331c,38107,1732098655202 in 233 msec 2024-11-20T10:31:12,339 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=8c533afd331c,38107,1732098655202 2024-11-20T10:31:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T10:31:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 8c533afd331c%2C41755%2C1732098655250.meta.1732098657196.meta, worker=8c533afd331c,38107,1732098655202 in 247 msec 2024-11-20T10:31:12,343 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting dir is empty, no logs to split. 2024-11-20T10:31:12,343 INFO [PEWorker-1 {}] master.SplitWALManager(105): 8c533afd331c,41755,1732098655250 WAL count=0, meta=true 2024-11-20T10:31:12,343 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 8c533afd331c,41755,1732098655250 WAL splitting is done? wals=0, meta=true 2024-11-20T10:31:12,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T10:31:12,346 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:31:12,347 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T10:31:12,409 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:12,410 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1] 2024-11-20T10:31:12,411 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8c533afd331c:41755 this server is in the failed servers list 2024-11-20T10:31:12,411 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8c533afd331c:41755 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8c533afd331c:41755 2024-11-20T10:31:12,411 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8c533afd331c:41755 2024-11-20T10:31:12,412 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 from cache 2024-11-20T10:31:12,497 DEBUG [8c533afd331c:38325 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T10:31:12,498 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(204): Hosts are {8c533afd331c=0} racks are {/default-rack=0} 2024-11-20T10:31:12,499 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T10:31:12,499 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T10:31:12,499 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T10:31:12,499 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T10:31:12,500 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T10:31:12,500 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T10:31:12,500 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T10:31:12,501 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8c533afd331c,38107,1732098655202 2024-11-20T10:31:12,503 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,38107,1732098655202, state=OPENING 2024-11-20T10:31:12,667 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:12,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:12,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:12,667 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T10:31:12,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:12,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:12,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:12,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,38107,1732098655202}] 2024-11-20T10:31:12,829 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T10:31:12,829 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:12,829 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T10:31:12,832 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8c533afd331c%2C38107%2C1732098655202.meta, suffix=.meta, logDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202, archiveDir=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs, maxLogs=32 2024-11-20T10:31:12,846 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202/8c533afd331c%2C38107%2C1732098655202.meta.1732098672832.meta, exclude list is [], retry=0 2024-11-20T10:31:12,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:12,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:12,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:12,851 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,38107,1732098655202/8c533afd331c%2C38107%2C1732098655202.meta.1732098672832.meta 2024-11-20T10:31:12,851 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:12,851 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:12,851 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T10:31:12,852 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T10:31:12,852 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T10:31:12,852 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T10:31:12,852 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:12,852 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T10:31:12,852 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T10:31:12,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T10:31:12,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T10:31:12,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:12,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:31:12,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T10:31:12,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T10:31:12,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:12,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:31:12,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T10:31:12,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T10:31:12,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:12,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:31:12,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T10:31:12,859 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T10:31:12,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:12,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T10:31:12,859 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T10:31:12,860 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:31:12,862 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740 2024-11-20T10:31:12,862 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T10:31:12,867 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T10:31:12,867 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-20T10:31:12,886 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/5839a4d6445743029938310f93f94ef6 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4./info:regioninfo/1732098671696/Put/seqid=0 2024-11-20T10:31:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741894_1072 (size=11177) 2024-11-20T10:31:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741894_1072 (size=11177) 2024-11-20T10:31:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741894_1072 (size=11177) 2024-11-20T10:31:12,893 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/5839a4d6445743029938310f93f94ef6 2024-11-20T10:31:12,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:12,920 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/ns/f5f4d5ef8c3641d9bab63fb526659b5d is 43, key is default/ns:d/1732098657369/Put/seqid=0 2024-11-20T10:31:12,921 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1] 2024-11-20T10:31:12,921 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:31:12,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:31:12,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38107 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:50608 deadline: 1732098732923, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 8c533afd331c,38107,1732098655202 2024-11-20T10:31:12,925 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 8c533afd331c,38107,1732098655202 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:31:12,925 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 8c533afd331c,38107,1732098655202 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T10:31:12,925 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1 from cache 2024-11-20T10:31:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741895_1073 (size=5153) 2024-11-20T10:31:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741895_1073 (size=5153) 2024-11-20T10:31:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741895_1073 (size=5153) 2024-11-20T10:31:12,930 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/ns/f5f4d5ef8c3641d9bab63fb526659b5d 2024-11-20T10:31:12,949 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/table/67554fbd71554838b3cbcf6a54d0bb00 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732098669963/Put/seqid=0 2024-11-20T10:31:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741896_1074 (size=5431) 2024-11-20T10:31:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741896_1074 (size=5431) 2024-11-20T10:31:12,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741896_1074 (size=5431) 2024-11-20T10:31:12,958 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/table/67554fbd71554838b3cbcf6a54d0bb00 2024-11-20T10:31:12,964 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/5839a4d6445743029938310f93f94ef6 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/info/5839a4d6445743029938310f93f94ef6 2024-11-20T10:31:12,971 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/info/5839a4d6445743029938310f93f94ef6, entries=36, sequenceid=18, filesize=10.9 K 2024-11-20T10:31:12,972 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/ns/f5f4d5ef8c3641d9bab63fb526659b5d as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/ns/f5f4d5ef8c3641d9bab63fb526659b5d 2024-11-20T10:31:12,978 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/ns/f5f4d5ef8c3641d9bab63fb526659b5d, entries=2, sequenceid=18, filesize=5.0 K 2024-11-20T10:31:12,979 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/table/67554fbd71554838b3cbcf6a54d0bb00 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/table/67554fbd71554838b3cbcf6a54d0bb00 2024-11-20T10:31:12,985 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/table/67554fbd71554838b3cbcf6a54d0bb00, entries=2, sequenceid=18, filesize=5.3 K 2024-11-20T10:31:12,986 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=18, compaction requested=false; wal=null 2024-11-20T10:31:12,986 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T10:31:12,988 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T10:31:12,988 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T10:31:12,989 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T10:31:12,991 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T10:31:12,994 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T10:31:12,995 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64865569, jitterRate=-0.03342770040035248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T10:31:12,995 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T10:31:12,996 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732098672852Writing region info on filesystem at 1732098672852Initializing all the Stores at 1732098672853 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098672853Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098672853Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098672853Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732098672853Obtaining lock to block concurrent updates at 1732098672867 (+14 ms)Preparing flush snapshotting stores in 1588230740 at 1732098672867Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732098672867Flushing stores of hbase:meta,,1.1588230740 at 1732098672867Flushing 1588230740/info: creating writer at 1732098672868 (+1 ms)Flushing 1588230740/info: appending metadata at 1732098672885 (+17 ms)Flushing 1588230740/info: closing flushed file at 1732098672885Flushing 1588230740/ns: creating writer at 1732098672900 (+15 ms)Flushing 1588230740/ns: appending metadata at 1732098672920 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732098672920Flushing 1588230740/table: creating writer at 1732098672935 (+15 ms)Flushing 1588230740/table: appending metadata at 1732098672949 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732098672949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19a8aeb3: reopening flushed file at 1732098672964 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f4c548: reopening flushed file at 1732098672971 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f16d033: reopening flushed file at 1732098672978 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=18, compaction requested=false; wal=null at 1732098672986 (+8 ms)Cleaning up temporary data from old regions at 1732098672988 (+2 ms)Running coprocessor post-open hooks at 1732098672995 (+7 ms)Region opened successfully at 1732098672996 (+1 ms) 2024-11-20T10:31:12,997 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732098672820 2024-11-20T10:31:13,000 DEBUG [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T10:31:13,000 INFO [RS_OPEN_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T10:31:13,000 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=8c533afd331c,38107,1732098655202 2024-11-20T10:31:13,001 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8c533afd331c,38107,1732098655202, state=OPEN 2024-11-20T10:31:13,022 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38325 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=8c533afd331c,38107,1732098655202, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T10:31:13,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:13,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:13,046 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T10:31:13,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:13,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:13,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T10:31:13,046 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8c533afd331c,38107,1732098655202 2024-11-20T10:31:13,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T10:31:13,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8c533afd331c,38107,1732098655202 in 379 msec 2024-11-20T10:31:13,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-20T10:31:13,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 704 msec 2024-11-20T10:31:13,053 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 8c533afd331c,41755,1732098655250 had 2 regions 2024-11-20T10:31:13,054 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 8c533afd331c,41755,1732098655250, splitWal=true, meta=true, isMeta: false 2024-11-20T10:31:13,056 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8c533afd331c,41755,1732098655250 WAL count=1, meta=false 2024-11-20T10:31:13,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 8c533afd331c%2C41755%2C1732098655250.1732098656718}] 2024-11-20T10:31:13,059 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=8c533afd331c,40989,1732098655109 2024-11-20T10:31:13,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 8c533afd331c%2C41755%2C1732098655250.1732098656718, worker=8c533afd331c,40989,1732098655109}] 2024-11-20T10:31:13,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40989 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-20T10:31:13,241 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718, size=2.1 K (2142bytes) 2024-11-20T10:31:13,242 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 2024-11-20T10:31:13,242 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 after 0ms 2024-11-20T10:31:13,245 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 took 4ms 2024-11-20T10:31:13,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 73ab4c4b40d6b3373ca57987a514eca4: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-20T10:31:13,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 so closing down 2024-11-20T10:31:13,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:13,248 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:13,248 INFO [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-20T10:31:13,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718, journal: Splitting hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718, size=2.1 K (2142bytes) at 1732098673242Finishing writing output for hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 so closing down at 1732098673247 (+5 ms)3 split writer threads finished at 1732098673248 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732098673248 2024-11-20T10:31:13,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 2024-11-20T10:31:13,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8c533afd331c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-20T10:31:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38325 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-20T10:31:13,252 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting/8c533afd331c%2C41755%2C1732098655250.1732098656718 to hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs 2024-11-20T10:31:13,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-20T10:31:13,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 8c533afd331c%2C41755%2C1732098655250.1732098656718, worker=8c533afd331c,40989,1732098655109 in 193 msec 2024-11-20T10:31:13,256 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=8c533afd331c,40989,1732098655109 2024-11-20T10:31:13,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-20T10:31:13,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 8c533afd331c%2C41755%2C1732098655250.1732098656718, worker=8c533afd331c,40989,1732098655109 in 200 msec 2024-11-20T10:31:13,260 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/WALs/8c533afd331c,41755,1732098655250-splitting dir is empty, no logs to split. 2024-11-20T10:31:13,260 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8c533afd331c,41755,1732098655250 WAL count=0, meta=false 2024-11-20T10:31:13,260 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 8c533afd331c,41755,1732098655250 WAL splitting is done? wals=0, meta=false 2024-11-20T10:31:13,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN}] 2024-11-20T10:31:13,263 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN 2024-11-20T10:31:13,264 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T10:31:13,415 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(204): Hosts are {8c533afd331c=0} racks are {/default-rack=0} 2024-11-20T10:31:13,415 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T10:31:13,415 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T10:31:13,415 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T10:31:13,416 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T10:31:13,416 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T10:31:13,416 INFO [8c533afd331c:38325 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T10:31:13,416 DEBUG [8c533afd331c:38325 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T10:31:13,417 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPENING, regionLocation=8c533afd331c,40989,1732098655109 2024-11-20T10:31:13,419 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8c533afd331c:41755 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8c533afd331c/172.17.0.2:41755 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:13,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1, error=java.net.ConnectException: Call to address=8c533afd331c:41755 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8c533afd331c/172.17.0.2:41755 2024-11-20T10:31:13,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T10:31:13,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8c533afd331c,41755,1732098655250, seqNum=-1 from cache 2024-11-20T10:31:13,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 8c533afd331c:41755 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8c533afd331c/172.17.0.2:41755 2024-11-20T10:31:13,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:13,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1] 2024-11-20T10:31:13,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T10:31:13,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T10:31:13,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN because future has completed 2024-11-20T10:31:13,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109}] 2024-11-20T10:31:13,725 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:13,725 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ab4c4b40d6b3373ca57987a514eca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:13,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,726 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:13,727 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,727 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,729 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,730 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf1 2024-11-20T10:31:13,730 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:13,737 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf1/f243daf9d2264427a6f57c43eb5ffa24 2024-11-20T10:31:13,738 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:13,738 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,739 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ab4c4b40d6b3373ca57987a514eca4 columnFamilyName cf2 2024-11-20T10:31:13,739 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:13,745 DEBUG [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/cf2/861b8e841b484ae1835cc5e0d2cfd2f1 2024-11-20T10:31:13,745 INFO [StoreOpener-73ab4c4b40d6b3373ca57987a514eca4-1 {}] regionserver.HStore(327): Store=73ab4c4b40d6b3373ca57987a514eca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:13,745 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,746 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,747 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,748 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,748 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,748 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T10:31:13,750 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,751 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 73ab4c4b40d6b3373ca57987a514eca4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68354496, jitterRate=0.018561363220214844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T10:31:13,751 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:13,751 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 73ab4c4b40d6b3373ca57987a514eca4: Running coprocessor pre-open hook at 1732098673727Writing region info on filesystem at 1732098673727Initializing all the Stores at 1732098673728 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098673728Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098673729 (+1 ms)Cleaning up temporary data from old regions at 1732098673748 (+19 ms)Running coprocessor post-open hooks at 1732098673751 (+3 ms)Region opened successfully at 1732098673751 2024-11-20T10:31:13,752 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., pid=21, masterSystemTime=1732098673716 2024-11-20T10:31:13,754 DEBUG [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:13,754 INFO [RS_OPEN_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:13,755 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=73ab4c4b40d6b3373ca57987a514eca4, regionState=OPEN, openSeqNum=18, regionLocation=8c533afd331c,40989,1732098655109 2024-11-20T10:31:13,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 because future has completed 2024-11-20T10:31:13,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T10:31:13,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ab4c4b40d6b3373ca57987a514eca4, server=8c533afd331c,40989,1732098655109 in 225 msec 2024-11-20T10:31:13,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-20T10:31:13,763 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 8c533afd331c,41755,1732098655250 after splitting done 2024-11-20T10:31:13,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=73ab4c4b40d6b3373ca57987a514eca4, ASSIGN in 498 msec 2024-11-20T10:31:13,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 8c533afd331c,41755,1732098655250, splitWal=true, meta=true in 1.6840 sec 2024-11-20T10:31:13,929 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T10:31:13,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8c533afd331c,38107,1732098655202, seqNum=-1] 2024-11-20T10:31:14,049 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8c533afd331c:41755 this server is in the failed servers list 2024-11-20T10:31:14,050 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8c533afd331c:41755 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8c533afd331c:41755 2024-11-20T10:31:14,050 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8c533afd331c:41755 2024-11-20T10:31:14,050 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,41755,1732098655250, seqNum=12 from cache 2024-11-20T10:31:14,268 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4., hostname=8c533afd331c,40989,1732098655109, seqNum=18] 2024-11-20T10:31:14,281 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 409), OpenFileDescriptor=1048 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 297), ProcessCount=11 (was 11), AvailableMemoryMB=8310 (was 8375) 2024-11-20T10:31:14,282 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-20T10:31:14,292 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=8307 2024-11-20T10:31:14,293 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-20T10:31:14,307 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:14,309 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:14,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:14,312 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61973613, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-61973613, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:14,325 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61973613/hregion-61973613.1732098674313, exclude list is [], retry=0 2024-11-20T10:31:14,328 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:14,328 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:14,329 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:14,332 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61973613/hregion-61973613.1732098674313 2024-11-20T10:31:14,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:14,332 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 690dc534bd31b75e4b01110f24abb070, NAME => 'testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741898_1076 (size=67) 2024-11-20T10:31:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741898_1076 (size=67) 2024-11-20T10:31:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741898_1076 (size=67) 2024-11-20T10:31:14,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T10:31:14,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:14,681 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-20T10:31:14,681 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:14,744 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:14,747 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,750 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName a 2024-11-20T10:31:14,750 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,751 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,751 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,753 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName b 2024-11-20T10:31:14,753 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,753 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,753 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,755 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName c 2024-11-20T10:31:14,755 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,755 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,755 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,758 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,758 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,758 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:14,759 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,762 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:14,762 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690dc534bd31b75e4b01110f24abb070; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66040488, jitterRate=-0.015920042991638184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690dc534bd31b75e4b01110f24abb070: Writing region info on filesystem at 1732098674745Initializing all the Stores at 1732098674746 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674746Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674746Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674747 (+1 ms)Cleaning up temporary data from old regions at 1732098674758 (+11 ms)Region opened successfully at 1732098674763 (+5 ms) 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 690dc534bd31b75e4b01110f24abb070, disabling compactions & flushes 2024-11-20T10:31:14,763 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. after waiting 0 ms 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,763 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,763 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 690dc534bd31b75e4b01110f24abb070: Waiting for close lock at 1732098674763Disabling compacts and flushes for region at 1732098674763Disabling writes for close at 1732098674763Writing region close event to WAL at 1732098674763Closed at 1732098674763 2024-11-20T10:31:14,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741897_1075 (size=93) 2024-11-20T10:31:14,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741897_1075 (size=93) 2024-11-20T10:31:14,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741897_1075 (size=93) 2024-11-20T10:31:14,768 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:14,768 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61973613:(num 1732098674313) 2024-11-20T10:31:14,768 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:14,769 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:14,781 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, exclude list is [], retry=0 2024-11-20T10:31:14,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:14,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:14,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:14,786 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 2024-11-20T10:31:14,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:14,787 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 690dc534bd31b75e4b01110f24abb070, NAME => 'testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:14,787 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:14,787 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,788 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,789 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,790 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName a 2024-11-20T10:31:14,790 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,790 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,790 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,791 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName b 2024-11-20T10:31:14,791 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,792 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,792 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,792 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName c 2024-11-20T10:31:14,792 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:14,793 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:14,793 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,794 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,795 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,795 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,796 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:14,797 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:14,797 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690dc534bd31b75e4b01110f24abb070; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73157354, jitterRate=0.09012952446937561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:14,798 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690dc534bd31b75e4b01110f24abb070: Writing region info on filesystem at 1732098674788Initializing all the Stores at 1732098674788Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674789 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674789Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098674789Cleaning up temporary data from old regions at 1732098674795 (+6 ms)Region opened successfully at 1732098674798 (+3 ms) 2024-11-20T10:31:14,820 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 690dc534bd31b75e4b01110f24abb070 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T10:31:14,824 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T10:31:14,836 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/a/37f0274097b446a88be2a7e150320071 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732098674798/Put/seqid=0 2024-11-20T10:31:14,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741900_1078 (size=5958) 2024-11-20T10:31:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741900_1078 (size=5958) 2024-11-20T10:31:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741900_1078 (size=5958) 2024-11-20T10:31:14,843 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/a/37f0274097b446a88be2a7e150320071 2024-11-20T10:31:14,861 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/6d5816eb2d0644a0aaa01424602a9960 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732098674806/Put/seqid=0 2024-11-20T10:31:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741901_1079 (size=5958) 2024-11-20T10:31:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741901_1079 (size=5958) 2024-11-20T10:31:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741901_1079 (size=5958) 2024-11-20T10:31:14,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/6d5816eb2d0644a0aaa01424602a9960 2024-11-20T10:31:14,886 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/c/eb1ba9bc01bb4bed9d656ff11b05ab83 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732098674812/Put/seqid=0 2024-11-20T10:31:14,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741902_1080 (size=5958) 2024-11-20T10:31:14,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741902_1080 (size=5958) 2024-11-20T10:31:14,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741902_1080 (size=5958) 2024-11-20T10:31:14,893 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/c/eb1ba9bc01bb4bed9d656ff11b05ab83 2024-11-20T10:31:14,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/a/37f0274097b446a88be2a7e150320071 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/a/37f0274097b446a88be2a7e150320071 2024-11-20T10:31:14,904 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/a/37f0274097b446a88be2a7e150320071, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T10:31:14,905 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/6d5816eb2d0644a0aaa01424602a9960 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/6d5816eb2d0644a0aaa01424602a9960 2024-11-20T10:31:14,910 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/6d5816eb2d0644a0aaa01424602a9960, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T10:31:14,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/c/eb1ba9bc01bb4bed9d656ff11b05ab83 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/c/eb1ba9bc01bb4bed9d656ff11b05ab83 2024-11-20T10:31:14,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/c/eb1ba9bc01bb4bed9d656ff11b05ab83, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T10:31:14,919 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 690dc534bd31b75e4b01110f24abb070 in 99ms, sequenceid=33, compaction requested=false 2024-11-20T10:31:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 690dc534bd31b75e4b01110f24abb070: 2024-11-20T10:31:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 690dc534bd31b75e4b01110f24abb070, disabling compactions & flushes 2024-11-20T10:31:14,919 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. after waiting 0 ms 2024-11-20T10:31:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,920 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. 2024-11-20T10:31:14,920 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 690dc534bd31b75e4b01110f24abb070: Waiting for close lock at 1732098674919Disabling compacts and flushes for region at 1732098674919Disabling writes for close at 1732098674919Writing region close event to WAL at 1732098674920 (+1 ms)Closed at 1732098674920 2024-11-20T10:31:14,924 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 not finished, retry = 0 2024-11-20T10:31:14,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741899_1077 (size=5937) 2024-11-20T10:31:14,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741899_1077 (size=5937) 2024-11-20T10:31:14,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741899_1077 (size=5937) 2024-11-20T10:31:15,029 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/6d5816eb2d0644a0aaa01424602a9960 to hdfs://localhost:36923/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/6d5816eb2d0644a0aaa01424602a9960 2024-11-20T10:31:15,052 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, size=5.8 K (5937bytes) 2024-11-20T10:31:15,052 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 2024-11-20T10:31:15,052 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 after 0ms 2024-11-20T10:31:15,055 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 took 4ms 2024-11-20T10:31:15,058 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 so closing down 2024-11-20T10:31:15,058 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:15,059 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732098674770.temp 2024-11-20T10:31:15,060 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp 2024-11-20T10:31:15,060 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741903_1081 (size=5223) 2024-11-20T10:31:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741903_1081 (size=5223) 2024-11-20T10:31:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741903_1081 (size=5223) 2024-11-20T10:31:15,068 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:15,070 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 2024-11-20T10:31:15,070 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-20T10:31:15,070 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, journal: Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, size=5.8 K (5937bytes) at 1732098675052Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 so closing down at 1732098675058 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp at 1732098675060 (+2 ms)3 split writer threads finished at 1732098675060Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732098675068 (+8 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000003-wal.1732098674770.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 at 1732098675070 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732098675070 2024-11-20T10:31:15,072 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098674770 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098674770 2024-11-20T10:31:15,073 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 2024-11-20T10:31:15,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:15,075 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:15,094 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098675076, exclude list is [], retry=0 2024-11-20T10:31:15,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:15,097 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:15,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:15,099 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098675076 2024-11-20T10:31:15,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:15,100 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 690dc534bd31b75e4b01110f24abb070, NAME => 'testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:15,100 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:15,100 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,100 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,102 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,103 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName a 2024-11-20T10:31:15,103 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,108 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/a/37f0274097b446a88be2a7e150320071 2024-11-20T10:31:15,108 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,109 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,110 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName b 2024-11-20T10:31:15,110 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,111 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,111 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,112 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690dc534bd31b75e4b01110f24abb070 columnFamilyName c 2024-11-20T10:31:15,112 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,118 DEBUG [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/c/eb1ba9bc01bb4bed9d656ff11b05ab83 2024-11-20T10:31:15,119 INFO [StoreOpener-690dc534bd31b75e4b01110f24abb070-1 {}] regionserver.HStore(327): Store=690dc534bd31b75e4b01110f24abb070/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,119 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,122 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 2024-11-20T10:31:15,125 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 2024-11-20T10:31:15,125 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 690dc534bd31b75e4b01110f24abb070 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T10:31:15,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/512c307228cf4d7f8c8113dcec974c5d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732098674806/Put/seqid=0 2024-11-20T10:31:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741905_1083 (size=5958) 2024-11-20T10:31:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741905_1083 (size=5958) 2024-11-20T10:31:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741905_1083 (size=5958) 2024-11-20T10:31:15,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/512c307228cf4d7f8c8113dcec974c5d 2024-11-20T10:31:15,151 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/.tmp/b/512c307228cf4d7f8c8113dcec974c5d as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/512c307228cf4d7f8c8113dcec974c5d 2024-11-20T10:31:15,156 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/b/512c307228cf4d7f8c8113dcec974c5d, entries=10, sequenceid=32, filesize=5.8 K 2024-11-20T10:31:15,156 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 690dc534bd31b75e4b01110f24abb070 in 31ms, sequenceid=32, compaction requested=false; wal=null 2024-11-20T10:31:15,157 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/0000000000000000032 2024-11-20T10:31:15,158 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,158 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,159 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:15,161 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690dc534bd31b75e4b01110f24abb070 2024-11-20T10:31:15,163 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/690dc534bd31b75e4b01110f24abb070/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-20T10:31:15,164 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690dc534bd31b75e4b01110f24abb070; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68775751, jitterRate=0.024838551878929138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:15,164 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690dc534bd31b75e4b01110f24abb070: Writing region info on filesystem at 1732098675100Initializing all the Stores at 1732098675101 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675101Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675102 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675102Obtaining lock to block concurrent updates at 1732098675125 (+23 ms)Preparing flush snapshotting stores in 690dc534bd31b75e4b01110f24abb070 at 1732098675125Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732098675125Flushing stores of testReplayEditsWrittenViaHRegion,,1732098674308.690dc534bd31b75e4b01110f24abb070. at 1732098675125Flushing 690dc534bd31b75e4b01110f24abb070/b: creating writer at 1732098675125Flushing 690dc534bd31b75e4b01110f24abb070/b: appending metadata at 1732098675138 (+13 ms)Flushing 690dc534bd31b75e4b01110f24abb070/b: closing flushed file at 1732098675138Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@394057b6: reopening flushed file at 1732098675150 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 690dc534bd31b75e4b01110f24abb070 in 31ms, sequenceid=32, compaction requested=false; wal=null at 1732098675156 (+6 ms)Cleaning up temporary data from old regions at 1732098675158 (+2 ms)Region opened successfully at 1732098675164 (+6 ms) 2024-11-20T10:31:15,186 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:58120 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:55078 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50494 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:35234 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:48316 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:37388 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 282), ProcessCount=11 (was 11), AvailableMemoryMB=8296 (was 8307) 2024-11-20T10:31:15,186 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-20T10:31:15,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=8296 2024-11-20T10:31:15,199 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-20T10:31:15,214 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:15,216 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:15,216 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:15,219 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01063008, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-01063008, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:15,232 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01063008/hregion-01063008.1732098675220, exclude list is [], retry=0 2024-11-20T10:31:15,235 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:15,236 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:15,237 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:15,238 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01063008/hregion-01063008.1732098675220 2024-11-20T10:31:15,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:15,239 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bd931baa84a2bdf997051fd724778156, NAME => 'testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741907_1085 (size=68) 2024-11-20T10:31:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741907_1085 (size=68) 2024-11-20T10:31:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741907_1085 (size=68) 2024-11-20T10:31:15,253 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:15,254 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,256 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName a 2024-11-20T10:31:15,256 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,257 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,257 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,259 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName b 2024-11-20T10:31:15,259 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,260 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,260 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,261 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName c 2024-11-20T10:31:15,262 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,262 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,266 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:15,267 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,270 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:15,270 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd931baa84a2bdf997051fd724778156; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74513896, jitterRate=0.1103435754776001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:15,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd931baa84a2bdf997051fd724778156: Writing region info on filesystem at 1732098675253Initializing all the Stores at 1732098675254 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675254Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675254Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675254Cleaning up temporary data from old regions at 1732098675265 (+11 ms)Region opened successfully at 1732098675271 (+6 ms) 2024-11-20T10:31:15,271 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bd931baa84a2bdf997051fd724778156, disabling compactions & flushes 2024-11-20T10:31:15,271 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:15,271 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:15,272 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. after waiting 1 ms 2024-11-20T10:31:15,272 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:15,272 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:15,272 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bd931baa84a2bdf997051fd724778156: Waiting for close lock at 1732098675271Disabling compacts and flushes for region at 1732098675271Disabling writes for close at 1732098675272 (+1 ms)Writing region close event to WAL at 1732098675272Closed at 1732098675272 2024-11-20T10:31:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741906_1084 (size=93) 2024-11-20T10:31:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741906_1084 (size=93) 2024-11-20T10:31:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741906_1084 (size=93) 2024-11-20T10:31:15,276 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-01063008/hregion-01063008.1732098675220 not finished, retry = 0 2024-11-20T10:31:15,379 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:15,379 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01063008:(num 1732098675220) 2024-11-20T10:31:15,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:15,381 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:15,400 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, exclude list is [], retry=0 2024-11-20T10:31:15,402 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:15,402 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:15,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:15,404 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 2024-11-20T10:31:15,404 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:15,460 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bd931baa84a2bdf997051fd724778156, NAME => 'testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:15,461 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:31:15,463 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,463 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:15,463 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,463 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,466 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,467 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName a 2024-11-20T10:31:15,467 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,468 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,468 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,470 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName b 2024-11-20T10:31:15,470 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,470 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,470 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,471 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName c 2024-11-20T10:31:15,471 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:15,472 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:15,472 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,474 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,475 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,475 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,475 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:15,476 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd931baa84a2bdf997051fd724778156; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62273275, jitterRate=-0.0720558911561966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:15,477 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:15,478 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd931baa84a2bdf997051fd724778156: Running coprocessor pre-open hook at 1732098675463Writing region info on filesystem at 1732098675463Initializing all the Stores at 1732098675465 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675465Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675465Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098675465Cleaning up temporary data from old regions at 1732098675475 (+10 ms)Running coprocessor post-open hooks at 1732098675477 (+2 ms)Region opened successfully at 1732098675478 (+1 ms) 2024-11-20T10:31:15,496 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd931baa84a2bdf997051fd724778156 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-20T10:31:15,497 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:16,498 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:17,500 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:18,501 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:19,502 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:20,504 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:20,967 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:31:21,505 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:22,507 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:23,508 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:23,515 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T10:31:24,509 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd931baa84a2bdf997051fd724778156/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:24,511 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd931baa84a2bdf997051fd724778156: 2024-11-20T10:31:24,511 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,616 INFO [AsyncFSWAL-0-hdfs://localhost:36923/hbase-prefix:default {}] wal.AbstractFSWAL(1368): Slow sync cost: 100 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK], DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK], DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK]] 2024-11-20T10:31:24,631 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd931baa84a2bdf997051fd724778156: 2024-11-20T10:31:24,632 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-20T10:31:24,632 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bd931baa84a2bdf997051fd724778156, disabling compactions & flushes 2024-11-20T10:31:24,632 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,632 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,632 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. after waiting 0 ms 2024-11-20T10:31:24,632 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,632 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,633 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. 2024-11-20T10:31:24,633 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bd931baa84a2bdf997051fd724778156: Waiting for close lock at 1732098684632Running coprocessor pre-close hooks at 1732098684632Disabling compacts and flushes for region at 1732098684632Disabling writes for close at 1732098684632Writing region close event to WAL at 1732098684632Running coprocessor post-close hooks at 1732098684632Closed at 1732098684633 (+1 ms) 2024-11-20T10:31:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741908_1086 (size=3941) 2024-11-20T10:31:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741908_1086 (size=3941) 2024-11-20T10:31:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741908_1086 (size=3941) 2024-11-20T10:31:24,652 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, size=3.8 K (3941bytes) 2024-11-20T10:31:24,652 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 2024-11-20T10:31:24,653 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 after 1ms 2024-11-20T10:31:24,655 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 took 3ms 2024-11-20T10:31:24,657 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 so closing down 2024-11-20T10:31:24,657 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:24,658 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732098675382.temp 2024-11-20T10:31:24,659 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp 2024-11-20T10:31:24,660 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741909_1087 (size=2983) 2024-11-20T10:31:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741909_1087 (size=2983) 2024-11-20T10:31:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741909_1087 (size=2983) 2024-11-20T10:31:24,666 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:24,667 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 2024-11-20T10:31:24,667 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-20T10:31:24,667 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, journal: Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, size=3.8 K (3941bytes) at 1732098684652Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 so closing down at 1732098684657 (+5 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp at 1732098684659 (+2 ms)3 split writer threads finished at 1732098684660 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732098684666 (+6 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000004-wal.1732098675382.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 at 1732098684667 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732098684667 2024-11-20T10:31:24,669 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098675382 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098675382 2024-11-20T10:31:24,670 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 2024-11-20T10:31:24,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:24,671 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:24,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-20T10:31:24,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:24,687 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098684672, exclude list is [], retry=0 2024-11-20T10:31:24,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:24,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:24,691 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:24,692 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098684672 2024-11-20T10:31:24,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:24,695 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bd931baa84a2bdf997051fd724778156, NAME => 'testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:24,695 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,695 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:24,696 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,696 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,697 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,698 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName a 2024-11-20T10:31:24,698 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:24,699 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:24,699 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,700 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName b 2024-11-20T10:31:24,700 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:24,700 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:24,700 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,701 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd931baa84a2bdf997051fd724778156 columnFamilyName c 2024-11-20T10:31:24,701 DEBUG [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:24,702 INFO [StoreOpener-bd931baa84a2bdf997051fd724778156-1 {}] regionserver.HStore(327): Store=bd931baa84a2bdf997051fd724778156/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:24,702 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,705 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 2024-11-20T10:31:24,708 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 2024-11-20T10:31:24,708 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd931baa84a2bdf997051fd724778156 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-20T10:31:24,725 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/a/c0de8bfc486244a88f500dea34205b4c is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732098684617/Put/seqid=0 2024-11-20T10:31:24,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741911_1089 (size=5523) 2024-11-20T10:31:24,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741911_1089 (size=5523) 2024-11-20T10:31:24,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741911_1089 (size=5523) 2024-11-20T10:31:24,731 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/a/c0de8bfc486244a88f500dea34205b4c 2024-11-20T10:31:24,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/b/4c49c0bfa3e441cab0a07ff0c6e024e9 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732098684512/Put/seqid=0 2024-11-20T10:31:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741912_1090 (size=5524) 2024-11-20T10:31:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741912_1090 (size=5524) 2024-11-20T10:31:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741912_1090 (size=5524) 2024-11-20T10:31:24,758 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/b/4c49c0bfa3e441cab0a07ff0c6e024e9 2024-11-20T10:31:24,776 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/c/e3e0bcc175294aeeb3660b58e3fae557 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732098684515/Put/seqid=0 2024-11-20T10:31:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741913_1091 (size=5457) 2024-11-20T10:31:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741913_1091 (size=5457) 2024-11-20T10:31:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741913_1091 (size=5457) 2024-11-20T10:31:24,784 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/c/e3e0bcc175294aeeb3660b58e3fae557 2024-11-20T10:31:24,790 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/a/c0de8bfc486244a88f500dea34205b4c as hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/a/c0de8bfc486244a88f500dea34205b4c 2024-11-20T10:31:24,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/a/c0de8bfc486244a88f500dea34205b4c, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T10:31:24,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/b/4c49c0bfa3e441cab0a07ff0c6e024e9 as hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/b/4c49c0bfa3e441cab0a07ff0c6e024e9 2024-11-20T10:31:24,801 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/b/4c49c0bfa3e441cab0a07ff0c6e024e9, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T10:31:24,802 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/.tmp/c/e3e0bcc175294aeeb3660b58e3fae557 as hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/c/e3e0bcc175294aeeb3660b58e3fae557 2024-11-20T10:31:24,808 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/c/e3e0bcc175294aeeb3660b58e3fae557, entries=6, sequenceid=26, filesize=5.3 K 2024-11-20T10:31:24,808 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bd931baa84a2bdf997051fd724778156 in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-11-20T10:31:24,809 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/0000000000000000026 2024-11-20T10:31:24,811 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,811 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,811 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:24,813 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,815 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsAfterAbortingFlush/bd931baa84a2bdf997051fd724778156/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-20T10:31:24,816 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd931baa84a2bdf997051fd724778156; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66526507, jitterRate=-0.00867779552936554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:24,816 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd931baa84a2bdf997051fd724778156 2024-11-20T10:31:24,817 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd931baa84a2bdf997051fd724778156: Running coprocessor pre-open hook at 1732098684696Writing region info on filesystem at 1732098684696Initializing all the Stores at 1732098684697 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098684697Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098684697Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098684697Obtaining lock to block concurrent updates at 1732098684708 (+11 ms)Preparing flush snapshotting stores in bd931baa84a2bdf997051fd724778156 at 1732098684708Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732098684708Flushing stores of testReplayEditsAfterAbortingFlush,,1732098675214.bd931baa84a2bdf997051fd724778156. at 1732098684708Flushing bd931baa84a2bdf997051fd724778156/a: creating writer at 1732098684709 (+1 ms)Flushing bd931baa84a2bdf997051fd724778156/a: appending metadata at 1732098684724 (+15 ms)Flushing bd931baa84a2bdf997051fd724778156/a: closing flushed file at 1732098684724Flushing bd931baa84a2bdf997051fd724778156/b: creating writer at 1732098684736 (+12 ms)Flushing bd931baa84a2bdf997051fd724778156/b: appending metadata at 1732098684750 (+14 ms)Flushing bd931baa84a2bdf997051fd724778156/b: closing flushed file at 1732098684750Flushing bd931baa84a2bdf997051fd724778156/c: creating writer at 1732098684762 (+12 ms)Flushing bd931baa84a2bdf997051fd724778156/c: appending metadata at 1732098684776 (+14 ms)Flushing bd931baa84a2bdf997051fd724778156/c: closing flushed file at 1732098684776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dcabb5a: reopening flushed file at 1732098684789 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b78492f: reopening flushed file at 1732098684795 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14b1b309: reopening flushed file at 1732098684801 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bd931baa84a2bdf997051fd724778156 in 100ms, sequenceid=26, compaction requested=false; wal=null at 1732098684808 (+7 ms)Cleaning up temporary data from old regions at 1732098684811 (+3 ms)Running coprocessor post-open hooks at 1732098684816 (+5 ms)Region opened successfully at 1732098684816 2024-11-20T10:31:24,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53348 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50984 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50644 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53376 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50670 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50970 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 282), ProcessCount=11 (was 11), AvailableMemoryMB=8269 (was 8296) 2024-11-20T10:31:24,836 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-20T10:31:24,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=8268 2024-11-20T10:31:24,851 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-20T10:31:24,872 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:24,874 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:24,875 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:24,878 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29884416, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-29884416, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:24,893 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29884416/hregion-29884416.1732098684879, exclude list is [], retry=0 2024-11-20T10:31:24,895 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:24,895 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:24,896 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:24,897 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29884416/hregion-29884416.1732098684879 2024-11-20T10:31:24,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:24,898 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d651d0da75f400ed668252e03bb083a3, NAME => 'testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741915_1093 (size=61) 2024-11-20T10:31:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741915_1093 (size=61) 2024-11-20T10:31:24,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741915_1093 (size=61) 2024-11-20T10:31:24,906 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:24,908 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,909 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d651d0da75f400ed668252e03bb083a3 columnFamilyName a 2024-11-20T10:31:24,909 DEBUG [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:24,909 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(327): Store=d651d0da75f400ed668252e03bb083a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:24,910 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,915 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:24,915 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d651d0da75f400ed668252e03bb083a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67327741, jitterRate=0.0032615214586257935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:31:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d651d0da75f400ed668252e03bb083a3: Writing region info on filesystem at 1732098684906Initializing all the Stores at 1732098684907 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098684907Cleaning up temporary data from old regions at 1732098684911 (+4 ms)Region opened successfully at 1732098684916 (+5 ms) 2024-11-20T10:31:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d651d0da75f400ed668252e03bb083a3, disabling compactions & flushes 2024-11-20T10:31:24,916 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. after waiting 0 ms 2024-11-20T10:31:24,916 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,917 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,917 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d651d0da75f400ed668252e03bb083a3: Waiting for close lock at 1732098684916Disabling compacts and flushes for region at 1732098684916Disabling writes for close at 1732098684916Writing region close event to WAL at 1732098684917 (+1 ms)Closed at 1732098684917 2024-11-20T10:31:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741914_1092 (size=93) 2024-11-20T10:31:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741914_1092 (size=93) 2024-11-20T10:31:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741914_1092 (size=93) 2024-11-20T10:31:24,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:24,921 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29884416:(num 1732098684879) 2024-11-20T10:31:24,921 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:24,923 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:24,936 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, exclude list is [], retry=0 2024-11-20T10:31:24,939 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:24,939 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:24,939 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:24,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 2024-11-20T10:31:24,941 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:24,941 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d651d0da75f400ed668252e03bb083a3, NAME => 'testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:24,941 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:24,941 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,941 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,943 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,943 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d651d0da75f400ed668252e03bb083a3 columnFamilyName a 2024-11-20T10:31:24,943 DEBUG [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:24,944 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(327): Store=d651d0da75f400ed668252e03bb083a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:24,944 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,946 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,946 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,946 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:24,949 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d651d0da75f400ed668252e03bb083a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62009449, jitterRate=-0.07598720490932465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:31:24,950 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d651d0da75f400ed668252e03bb083a3: Writing region info on filesystem at 1732098684941Initializing all the Stores at 1732098684942 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098684942Cleaning up temporary data from old regions at 1732098684947 (+5 ms)Region opened successfully at 1732098684950 (+3 ms) 2024-11-20T10:31:24,959 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d651d0da75f400ed668252e03bb083a3, disabling compactions & flushes 2024-11-20T10:31:24,959 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,959 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,959 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. after waiting 0 ms 2024-11-20T10:31:24,959 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,959 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,959 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. 2024-11-20T10:31:24,959 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d651d0da75f400ed668252e03bb083a3: Waiting for close lock at 1732098684959Disabling compacts and flushes for region at 1732098684959Disabling writes for close at 1732098684959Writing region close event to WAL at 1732098684959Closed at 1732098684959 2024-11-20T10:31:24,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741916_1094 (size=1623) 2024-11-20T10:31:24,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741916_1094 (size=1623) 2024-11-20T10:31:24,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741916_1094 (size=1623) 2024-11-20T10:31:24,978 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, size=1.6 K (1623bytes) 2024-11-20T10:31:24,978 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 2024-11-20T10:31:24,979 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 after 1ms 2024-11-20T10:31:24,981 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 took 3ms 2024-11-20T10:31:24,983 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 so closing down 2024-11-20T10:31:24,983 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:24,984 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732098684923.temp 2024-11-20T10:31:24,985 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp 2024-11-20T10:31:24,986 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741917_1095 (size=1623) 2024-11-20T10:31:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741917_1095 (size=1623) 2024-11-20T10:31:24,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741917_1095 (size=1623) 2024-11-20T10:31:24,992 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:24,994 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp to hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 2024-11-20T10:31:24,994 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-20T10:31:24,994 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, journal: Splitting hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, size=1.6 K (1623bytes) at 1732098684978Finishing writing output for hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 so closing down at 1732098684983 (+5 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp at 1732098684985 (+2 ms)3 split writer threads finished at 1732098684986 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732098684992 (+6 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000003-wal.1732098684923.temp to hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 at 1732098684994 (+2 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732098684994 2024-11-20T10:31:24,995 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098684923 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098684923 2024-11-20T10:31:24,996 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 2024-11-20T10:31:25,326 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:25,328 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:25,341 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098685329, exclude list is [], retry=0 2024-11-20T10:31:25,344 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:25,344 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:25,344 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:25,346 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098685329 2024-11-20T10:31:25,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:25,346 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d651d0da75f400ed668252e03bb083a3, NAME => 'testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:25,346 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:25,346 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,346 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,350 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,352 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d651d0da75f400ed668252e03bb083a3 columnFamilyName a 2024-11-20T10:31:25,352 DEBUG [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,353 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(327): Store=d651d0da75f400ed668252e03bb083a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,354 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,354 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,356 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,357 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 2024-11-20T10:31:25,360 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 2024-11-20T10:31:25,360 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d651d0da75f400ed668252e03bb083a3 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-20T10:31:25,377 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/.tmp/a/c25dfc8d23074cf2b222d74c76091979 is 79, key is testDatalossWhenInputError/a:x0/1732098684950/Put/seqid=0 2024-11-20T10:31:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741919_1097 (size=5808) 2024-11-20T10:31:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741919_1097 (size=5808) 2024-11-20T10:31:25,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741919_1097 (size=5808) 2024-11-20T10:31:25,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/.tmp/a/c25dfc8d23074cf2b222d74c76091979 2024-11-20T10:31:25,395 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/.tmp/a/c25dfc8d23074cf2b222d74c76091979 as hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/a/c25dfc8d23074cf2b222d74c76091979 2024-11-20T10:31:25,405 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/a/c25dfc8d23074cf2b222d74c76091979, entries=10, sequenceid=12, filesize=5.7 K 2024-11-20T10:31:25,405 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d651d0da75f400ed668252e03bb083a3 in 45ms, sequenceid=12, compaction requested=false; wal=null 2024-11-20T10:31:25,406 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/0000000000000000012 2024-11-20T10:31:25,407 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,407 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,411 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,413 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-20T10:31:25,414 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d651d0da75f400ed668252e03bb083a3; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61652855, jitterRate=-0.08130086958408356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:31:25,414 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d651d0da75f400ed668252e03bb083a3: Writing region info on filesystem at 1732098685346Initializing all the Stores at 1732098685349 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685349Obtaining lock to block concurrent updates at 1732098685360 (+11 ms)Preparing flush snapshotting stores in d651d0da75f400ed668252e03bb083a3 at 1732098685360Finished memstore snapshotting testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732098685360Flushing stores of testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3. at 1732098685360Flushing d651d0da75f400ed668252e03bb083a3/a: creating writer at 1732098685360Flushing d651d0da75f400ed668252e03bb083a3/a: appending metadata at 1732098685377 (+17 ms)Flushing d651d0da75f400ed668252e03bb083a3/a: closing flushed file at 1732098685377Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7244d4ec: reopening flushed file at 1732098685393 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d651d0da75f400ed668252e03bb083a3 in 45ms, sequenceid=12, compaction requested=false; wal=null at 1732098685405 (+12 ms)Cleaning up temporary data from old regions at 1732098685407 (+2 ms)Region opened successfully at 1732098685414 (+7 ms) 2024-11-20T10:31:25,417 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d651d0da75f400ed668252e03bb083a3, NAME => 'testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:25,417 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732098684873.d651d0da75f400ed668252e03bb083a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:25,417 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,417 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,418 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,419 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d651d0da75f400ed668252e03bb083a3 columnFamilyName a 2024-11-20T10:31:25,419 DEBUG [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,424 DEBUG [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/a/c25dfc8d23074cf2b222d74c76091979 2024-11-20T10:31:25,424 INFO [StoreOpener-d651d0da75f400ed668252e03bb083a3-1 {}] regionserver.HStore(327): Store=d651d0da75f400ed668252e03bb083a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,425 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,425 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,427 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,427 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,427 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,429 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d651d0da75f400ed668252e03bb083a3 2024-11-20T10:31:25,432 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testDatalossWhenInputError/d651d0da75f400ed668252e03bb083a3/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-20T10:31:25,433 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d651d0da75f400ed668252e03bb083a3; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68680791, jitterRate=0.023423537611961365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T10:31:25,434 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d651d0da75f400ed668252e03bb083a3: Writing region info on filesystem at 1732098685417Initializing all the Stores at 1732098685418 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685418Cleaning up temporary data from old regions at 1732098685427 (+9 ms)Region opened successfully at 1732098685433 (+6 ms) 2024-11-20T10:31:25,454 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=426 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50710 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53446 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53376 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:51050 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50670 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50970 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=8195 (was 8268) 2024-11-20T10:31:25,454 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-20T10:31:25,467 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=426, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=8193 2024-11-20T10:31:25,467 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-20T10:31:25,485 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:25,486 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:25,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:25,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29362329, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-29362329, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:25,503 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29362329/hregion-29362329.1732098685489, exclude list is [], retry=0 2024-11-20T10:31:25,506 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:25,506 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:25,506 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:25,508 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29362329/hregion-29362329.1732098685489 2024-11-20T10:31:25,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:25,510 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 047f2e9d7a931eb1f0305b1b9762e1e2, NAME => 'testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741921_1099 (size=63) 2024-11-20T10:31:25,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741921_1099 (size=63) 2024-11-20T10:31:25,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741921_1099 (size=63) 2024-11-20T10:31:25,543 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:25,544 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,545 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName a 2024-11-20T10:31:25,545 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,546 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,546 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,547 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName b 2024-11-20T10:31:25,547 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,548 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,548 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,549 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName c 2024-11-20T10:31:25,549 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,550 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,550 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,551 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,551 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,552 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,552 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,553 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:25,554 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:25,556 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 047f2e9d7a931eb1f0305b1b9762e1e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64005292, jitterRate=-0.04624682664871216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 047f2e9d7a931eb1f0305b1b9762e1e2: Writing region info on filesystem at 1732098685543Initializing all the Stores at 1732098685543Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685544 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685544Cleaning up temporary data from old regions at 1732098685552 (+8 ms)Region opened successfully at 1732098685557 (+5 ms) 2024-11-20T10:31:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 047f2e9d7a931eb1f0305b1b9762e1e2, disabling compactions & flushes 2024-11-20T10:31:25,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. after waiting 0 ms 2024-11-20T10:31:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:25,557 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:25,558 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 047f2e9d7a931eb1f0305b1b9762e1e2: Waiting for close lock at 1732098685557Disabling compacts and flushes for region at 1732098685557Disabling writes for close at 1732098685557Writing region close event to WAL at 1732098685557Closed at 1732098685557 2024-11-20T10:31:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741920_1098 (size=93) 2024-11-20T10:31:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741920_1098 (size=93) 2024-11-20T10:31:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741920_1098 (size=93) 2024-11-20T10:31:25,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:25,562 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29362329:(num 1732098685489) 2024-11-20T10:31:25,562 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:25,564 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:25,576 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, exclude list is [], retry=0 2024-11-20T10:31:25,578 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:25,578 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:25,578 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:25,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 2024-11-20T10:31:25,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:25,580 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 047f2e9d7a931eb1f0305b1b9762e1e2, NAME => 'testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:25,580 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:25,580 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,580 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,582 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,582 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName a 2024-11-20T10:31:25,582 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,583 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,583 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,584 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName b 2024-11-20T10:31:25,584 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,584 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,584 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,585 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName c 2024-11-20T10:31:25,585 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:25,585 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:25,585 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,586 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,587 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,587 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,588 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:25,589 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:25,589 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 047f2e9d7a931eb1f0305b1b9762e1e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59034928, jitterRate=-0.12031102180480957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:25,590 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 047f2e9d7a931eb1f0305b1b9762e1e2: Writing region info on filesystem at 1732098685580Initializing all the Stores at 1732098685581 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685581Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685582 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098685582Cleaning up temporary data from old regions at 1732098685587 (+5 ms)Region opened successfully at 1732098685590 (+3 ms) 2024-11-20T10:31:25,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732098685593/Put/seqid=0 2024-11-20T10:31:25,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741923_1101 (size=4875) 2024-11-20T10:31:25,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741923_1101 (size=4875) 2024-11-20T10:31:25,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741923_1101 (size=4875) 2024-11-20T10:31:25,603 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732098685603/Put/seqid=0 2024-11-20T10:31:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741924_1102 (size=4875) 2024-11-20T10:31:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741924_1102 (size=4875) 2024-11-20T10:31:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741924_1102 (size=4875) 2024-11-20T10:31:25,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732098685615/Put/seqid=0 2024-11-20T10:31:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741925_1103 (size=4875) 2024-11-20T10:31:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741925_1103 (size=4875) 2024-11-20T10:31:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741925_1103 (size=4875) 2024-11-20T10:31:25,622 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,625 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-20T10:31:25,625 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T10:31:25,625 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,628 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-20T10:31:25,628 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T10:31:25,629 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,631 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-20T10:31:25,631 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T10:31:25,631 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 047f2e9d7a931eb1f0305b1b9762e1e2 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-20T10:31:25,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp/a/542efd2c3a124f1fb6b57871823740b8 is 55, key is testCompactedBulkLoadedFiles/a:a/1732098685590/Put/seqid=0 2024-11-20T10:31:25,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741926_1104 (size=5107) 2024-11-20T10:31:25,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741926_1104 (size=5107) 2024-11-20T10:31:25,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741926_1104 (size=5107) 2024-11-20T10:31:25,653 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp/a/542efd2c3a124f1fb6b57871823740b8 2024-11-20T10:31:25,658 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp/a/542efd2c3a124f1fb6b57871823740b8 as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8 2024-11-20T10:31:25,663 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8, entries=1, sequenceid=4, filesize=5.0 K 2024-11-20T10:31:25,664 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 047f2e9d7a931eb1f0305b1b9762e1e2 in 33ms, sequenceid=4, compaction requested=false 2024-11-20T10:31:25,664 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 047f2e9d7a931eb1f0305b1b9762e1e2: 2024-11-20T10:31:25,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ 2024-11-20T10:31:25,667 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ 2024-11-20T10:31:25,668 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ 2024-11-20T10:31:25,669 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile0 into 047f2e9d7a931eb1f0305b1b9762e1e2/a as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ - updating store file list. 2024-11-20T10:31:25,674 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3908463f781c4d3aac4959ecc8f881a7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:25,674 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ into 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,674 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile0 into 047f2e9d7a931eb1f0305b1b9762e1e2/a (new location: hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_) 2024-11-20T10:31:25,676 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile1 into 047f2e9d7a931eb1f0305b1b9762e1e2/a as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ - updating store file list. 2024-11-20T10:31:25,681 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32d62db2e5964997b85b44851aa35e8b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:25,681 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ into 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,681 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile1 into 047f2e9d7a931eb1f0305b1b9762e1e2/a (new location: hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_) 2024-11-20T10:31:25,682 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile2 into 047f2e9d7a931eb1f0305b1b9762e1e2/a as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ - updating store file list. 2024-11-20T10:31:25,686 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 12650ece7a594c1b82e6d4727b916782_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:25,686 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ into 047f2e9d7a931eb1f0305b1b9762e1e2/a 2024-11-20T10:31:25,686 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36923/hbase/testCompactedBulkLoadedFiles/hfile2 into 047f2e9d7a931eb1f0305b1b9762e1e2/a (new location: hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_) 2024-11-20T10:31:25,693 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T10:31:25,693 DEBUG [Time-limited test {}] regionserver.HStore(1541): 047f2e9d7a931eb1f0305b1b9762e1e2/a is initiating major compaction (all files) 2024-11-20T10:31:25,693 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 047f2e9d7a931eb1f0305b1b9762e1e2/a in testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:25,693 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_] into tmpdir=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp, totalSize=19.3 K 2024-11-20T10:31:25,694 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 542efd2c3a124f1fb6b57871823740b8, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732098685590 2024-11-20T10:31:25,694 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3908463f781c4d3aac4959ecc8f881a7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T10:31:25,695 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 32d62db2e5964997b85b44851aa35e8b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T10:31:25,695 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 12650ece7a594c1b82e6d4727b916782_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T10:31:25,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp/a/d1a0fd3671c74e9f88dace4a7879080b is 55, key is testCompactedBulkLoadedFiles/a:a/1732098685590/Put/seqid=0 2024-11-20T10:31:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741927_1105 (size=6154) 2024-11-20T10:31:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741927_1105 (size=6154) 2024-11-20T10:31:25,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741927_1105 (size=6154) 2024-11-20T10:31:25,718 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/.tmp/a/d1a0fd3671c74e9f88dace4a7879080b as hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/d1a0fd3671c74e9f88dace4a7879080b 2024-11-20T10:31:25,724 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 047f2e9d7a931eb1f0305b1b9762e1e2/a of 047f2e9d7a931eb1f0305b1b9762e1e2 into d1a0fd3671c74e9f88dace4a7879080b(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T10:31:25,724 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 047f2e9d7a931eb1f0305b1b9762e1e2: 2024-11-20T10:31:25,724 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T10:31:25,724 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T10:31:25,755 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, size=0 (0bytes) 2024-11-20T10:31:25,755 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 might be still open, length is 0 2024-11-20T10:31:25,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 2024-11-20T10:31:25,756 WARN [IPC Server handler 1 on default port 36923 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-20T10:31:25,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 after 0ms 2024-11-20T10:31:26,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53476 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53476 dst: /127.0.0.1:39533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39533 remote=/127.0.0.1:53476]. Total timeout mills is 60000, 59302 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:26,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:51106 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:46093:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51106 dst: /127.0.0.1:46093 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:26,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50744 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50744 dst: /127.0.0.1:38347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:26,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741922_1106 (size=1659) 2024-11-20T10:31:26,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741922_1106 (size=1659) 2024-11-20T10:31:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741922_1106 (size=1659) 2024-11-20T10:31:29,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 after 4002ms 2024-11-20T10:31:29,816 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 took 4061ms 2024-11-20T10:31:29,821 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564; continuing. 2024-11-20T10:31:29,821 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 so closing down 2024-11-20T10:31:29,821 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:29,825 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732098685564.temp 2024-11-20T10:31:29,828 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp 2024-11-20T10:31:29,828 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741928_1107 (size=698) 2024-11-20T10:31:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741928_1107 (size=698) 2024-11-20T10:31:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741928_1107 (size=698) 2024-11-20T10:31:29,839 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:29,840 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp to hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 2024-11-20T10:31:29,840 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 22 ms; skipped=3; WAL=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T10:31:29,840 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, journal: Splitting hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, size=0 (0bytes) at 1732098685755Finishing writing output for hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 so closing down at 1732098689821 (+4066 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp at 1732098689828 (+7 ms)3 split writer threads finished at 1732098689828Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732098689839 (+11 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000003-wal.1732098685564.temp to hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 at 1732098689840 (+1 ms)Processed 5 edits across 1 Regions in 22 ms; skipped=3; WAL=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564, size=0, length=0, corrupted=false, cancelled=false at 1732098689840 2024-11-20T10:31:29,842 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098685564 2024-11-20T10:31:29,842 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 2024-11-20T10:31:29,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:29,844 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:29,856 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098689844, exclude list is [], retry=0 2024-11-20T10:31:29,858 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:29,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:29,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:29,860 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098689844 2024-11-20T10:31:29,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305)] 2024-11-20T10:31:29,861 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 047f2e9d7a931eb1f0305b1b9762e1e2, NAME => 'testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:29,861 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:29,861 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,861 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,862 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,863 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName a 2024-11-20T10:31:29,863 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:29,869 DEBUG [StoreFileOpener-047f2e9d7a931eb1f0305b1b9762e1e2-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 12650ece7a594c1b82e6d4727b916782_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:29,869 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ 2024-11-20T10:31:29,873 DEBUG [StoreFileOpener-047f2e9d7a931eb1f0305b1b9762e1e2-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32d62db2e5964997b85b44851aa35e8b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:29,873 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ 2024-11-20T10:31:29,877 DEBUG [StoreFileOpener-047f2e9d7a931eb1f0305b1b9762e1e2-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3908463f781c4d3aac4959ecc8f881a7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T10:31:29,877 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ 2024-11-20T10:31:29,881 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8 2024-11-20T10:31:29,885 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/d1a0fd3671c74e9f88dace4a7879080b 2024-11-20T10:31:29,885 WARN [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-11-20T10:31:29,885 WARN [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-11-20T10:31:29,885 WARN [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-11-20T10:31:29,885 WARN [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-11-20T10:31:29,885 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8] to archive 2024-11-20T10:31:29,886 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T10:31:29,888 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ to hdfs://localhost:36923/hbase/archive/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/12650ece7a594c1b82e6d4727b916782_SeqId_4_ 2024-11-20T10:31:29,889 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ to hdfs://localhost:36923/hbase/archive/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/32d62db2e5964997b85b44851aa35e8b_SeqId_4_ 2024-11-20T10:31:29,890 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ to hdfs://localhost:36923/hbase/archive/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/3908463f781c4d3aac4959ecc8f881a7_SeqId_4_ 2024-11-20T10:31:29,891 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8 to hdfs://localhost:36923/hbase/archive/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/a/542efd2c3a124f1fb6b57871823740b8 2024-11-20T10:31:29,891 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:29,891 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,892 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName b 2024-11-20T10:31:29,892 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:29,893 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:29,893 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,893 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 047f2e9d7a931eb1f0305b1b9762e1e2 columnFamilyName c 2024-11-20T10:31:29,893 DEBUG [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:29,894 INFO [StoreOpener-047f2e9d7a931eb1f0305b1b9762e1e2-1 {}] regionserver.HStore(327): Store=047f2e9d7a931eb1f0305b1b9762e1e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:29,894 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,895 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,896 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,897 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 2024-11-20T10:31:29,900 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 047f2e9d7a931eb1f0305b1b9762e1e2 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "047f2e9d7a931eb1f0305b1b9762e1e2" family_name: "a" compaction_input: "542efd2c3a124f1fb6b57871823740b8" compaction_input: "3908463f781c4d3aac4959ecc8f881a7_SeqId_4_" compaction_input: "32d62db2e5964997b85b44851aa35e8b_SeqId_4_" compaction_input: "12650ece7a594c1b82e6d4727b916782_SeqId_4_" compaction_output: "d1a0fd3671c74e9f88dace4a7879080b" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-20T10:31:29,901 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-20T10:31:29,901 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 2024-11-20T10:31:29,902 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/0000000000000000008 2024-11-20T10:31:29,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,904 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:29,905 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 047f2e9d7a931eb1f0305b1b9762e1e2 2024-11-20T10:31:29,907 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testCompactedBulkLoadedFiles/047f2e9d7a931eb1f0305b1b9762e1e2/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T10:31:29,908 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 047f2e9d7a931eb1f0305b1b9762e1e2; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65526003, jitterRate=-0.023586466908454895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:29,908 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 047f2e9d7a931eb1f0305b1b9762e1e2: Writing region info on filesystem at 1732098689861Initializing all the Stores at 1732098689862 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098689862Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098689862Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098689862Cleaning up temporary data from old regions at 1732098689903 (+41 ms)Region opened successfully at 1732098689908 (+5 ms) 2024-11-20T10:31:29,911 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 047f2e9d7a931eb1f0305b1b9762e1e2, disabling compactions & flushes 2024-11-20T10:31:29,911 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:29,911 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:29,911 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. after waiting 0 ms 2024-11-20T10:31:29,911 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:29,912 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732098685485.047f2e9d7a931eb1f0305b1b9762e1e2. 2024-11-20T10:31:29,912 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 047f2e9d7a931eb1f0305b1b9762e1e2: Waiting for close lock at 1732098689911Disabling compacts and flushes for region at 1732098689911Disabling writes for close at 1732098689911Writing region close event to WAL at 1732098689912 (+1 ms)Closed at 1732098689912 2024-11-20T10:31:29,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741929_1108 (size=93) 2024-11-20T10:31:29,914 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098689844 not finished, retry = 0 2024-11-20T10:31:29,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741929_1108 (size=93) 2024-11-20T10:31:29,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741929_1108 (size=93) 2024-11-20T10:31:30,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:30,017 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732098689844) 2024-11-20T10:31:30,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 426) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1226529806_22 at /127.0.0.1:50790 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36923 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1226529806_22 at /127.0.0.1:53530 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:36923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1226529806_22 at /127.0.0.1:51192 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=8120 (was 8193) 2024-11-20T10:31:30,034 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T10:31:30,046 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=8120 2024-11-20T10:31:30,046 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T10:31:30,061 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:30,062 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T10:31:30,063 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T10:31:30,065 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36311657, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/hregion-36311657, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:30,079 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36311657/hregion-36311657.1732098690066, exclude list is [], retry=0 2024-11-20T10:31:30,082 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:30,082 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:30,082 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:30,084 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36311657/hregion-36311657.1732098690066 2024-11-20T10:31:30,084 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:30,085 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ff56393222adf494822dde15e6906700, NAME => 'testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36923/hbase 2024-11-20T10:31:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741931_1110 (size=67) 2024-11-20T10:31:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741931_1110 (size=67) 2024-11-20T10:31:30,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741931_1110 (size=67) 2024-11-20T10:31:30,094 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:30,096 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,098 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName a 2024-11-20T10:31:30,098 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,098 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,099 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,100 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName b 2024-11-20T10:31:30,100 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,101 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,101 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,102 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName c 2024-11-20T10:31:30,102 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,103 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,103 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,105 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,105 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,106 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:30,107 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,109 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T10:31:30,110 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ff56393222adf494822dde15e6906700; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63409579, jitterRate=-0.05512364208698273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:30,110 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ff56393222adf494822dde15e6906700: Writing region info on filesystem at 1732098690095Initializing all the Stores at 1732098690095Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690095Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690096 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690096Cleaning up temporary data from old regions at 1732098690105 (+9 ms)Region opened successfully at 1732098690110 (+5 ms) 2024-11-20T10:31:30,110 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ff56393222adf494822dde15e6906700, disabling compactions & flushes 2024-11-20T10:31:30,110 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,110 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,110 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. after waiting 0 ms 2024-11-20T10:31:30,110 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,111 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,111 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ff56393222adf494822dde15e6906700: Waiting for close lock at 1732098690110Disabling compacts and flushes for region at 1732098690110Disabling writes for close at 1732098690110Writing region close event to WAL at 1732098690111 (+1 ms)Closed at 1732098690111 2024-11-20T10:31:30,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741930_1109 (size=93) 2024-11-20T10:31:30,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741930_1109 (size=93) 2024-11-20T10:31:30,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741930_1109 (size=93) 2024-11-20T10:31:30,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:30,115 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36311657:(num 1732098690066) 2024-11-20T10:31:30,115 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:30,117 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:30,129 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, exclude list is [], retry=0 2024-11-20T10:31:30,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:30,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:30,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:30,134 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 2024-11-20T10:31:30,134 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:30,134 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ff56393222adf494822dde15e6906700, NAME => 'testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:30,134 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:30,134 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,134 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,136 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,137 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName a 2024-11-20T10:31:30,137 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T10:31:30,137 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,137 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,137 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,138 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName b 2024-11-20T10:31:30,138 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,139 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,139 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,139 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName c 2024-11-20T10:31:30,139 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,140 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,140 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,141 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,142 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,143 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,143 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,144 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:30,145 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,146 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ff56393222adf494822dde15e6906700; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73195178, jitterRate=0.0906931459903717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:30,146 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ff56393222adf494822dde15e6906700: Writing region info on filesystem at 1732098690134Initializing all the Stores at 1732098690135 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690135Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690136 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690136Cleaning up temporary data from old regions at 1732098690143 (+7 ms)Region opened successfully at 1732098690146 (+3 ms) 2024-11-20T10:31:30,153 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ff56393222adf494822dde15e6906700 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T10:31:30,168 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/42a8ee940dba4261ab456a889bd73624 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732098690146/Put/seqid=0 2024-11-20T10:31:30,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741933_1112 (size=5958) 2024-11-20T10:31:30,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741933_1112 (size=5958) 2024-11-20T10:31:30,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741933_1112 (size=5958) 2024-11-20T10:31:30,175 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/42a8ee940dba4261ab456a889bd73624 2024-11-20T10:31:30,180 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/42a8ee940dba4261ab456a889bd73624 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/42a8ee940dba4261ab456a889bd73624 2024-11-20T10:31:30,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/42a8ee940dba4261ab456a889bd73624, entries=10, sequenceid=13, filesize=5.8 K 2024-11-20T10:31:30,186 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ff56393222adf494822dde15e6906700 in 33ms, sequenceid=13, compaction requested=false 2024-11-20T10:31:30,186 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ff56393222adf494822dde15e6906700: 2024-11-20T10:31:30,200 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ff56393222adf494822dde15e6906700, disabling compactions & flushes 2024-11-20T10:31:30,200 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,200 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,200 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. after waiting 0 ms 2024-11-20T10:31:30,200 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,201 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,201 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:30,201 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ff56393222adf494822dde15e6906700: Waiting for close lock at 1732098690200Disabling compacts and flushes for region at 1732098690200Disabling writes for close at 1732098690200Writing region close event to WAL at 1732098690201 (+1 ms)Closed at 1732098690201 2024-11-20T10:31:30,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741932_1111 (size=5869) 2024-11-20T10:31:30,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741932_1111 (size=5869) 2024-11-20T10:31:30,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741932_1111 (size=5869) 2024-11-20T10:31:30,225 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, size=5.7 K (5869bytes) 2024-11-20T10:31:30,225 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 2024-11-20T10:31:30,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 after 1ms 2024-11-20T10:31:30,228 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 took 3ms 2024-11-20T10:31:30,231 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 so closing down 2024-11-20T10:31:30,231 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:30,232 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732098690117.temp 2024-11-20T10:31:30,233 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp 2024-11-20T10:31:30,233 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741934_1113 (size=5223) 2024-11-20T10:31:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741934_1113 (size=5223) 2024-11-20T10:31:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741934_1113 (size=5223) 2024-11-20T10:31:30,239 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:30,240 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 2024-11-20T10:31:30,240 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-20T10:31:30,240 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, journal: Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, size=5.7 K (5869bytes) at 1732098690225Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 so closing down at 1732098690231 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp at 1732098690233 (+2 ms)3 split writer threads finished at 1732098690233Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732098690239 (+6 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000003-wal.1732098690117.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 at 1732098690240 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732098690240 2024-11-20T10:31:30,242 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690117 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098690117 2024-11-20T10:31:30,243 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 2024-11-20T10:31:30,243 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:30,244 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:30,264 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, exclude list is [], retry=0 2024-11-20T10:31:30,266 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:30,267 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:30,267 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:30,269 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 2024-11-20T10:31:30,269 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41233:41233),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:34509:34509)] 2024-11-20T10:31:30,269 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ff56393222adf494822dde15e6906700, NAME => 'testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.', STARTKEY => '', ENDKEY => ''} 2024-11-20T10:31:30,269 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:30,270 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,270 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,271 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,272 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName a 2024-11-20T10:31:30,272 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,278 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/42a8ee940dba4261ab456a889bd73624 2024-11-20T10:31:30,278 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,278 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,279 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName b 2024-11-20T10:31:30,279 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,280 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,280 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,281 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName c 2024-11-20T10:31:30,281 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:30,281 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:30,281 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,282 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,284 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 2024-11-20T10:31:30,287 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 2024-11-20T10:31:30,287 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ff56393222adf494822dde15e6906700 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-20T10:31:30,301 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e61a42b0ace64281bd862b0c973ae9a7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732098690186/Put/seqid=0 2024-11-20T10:31:30,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741936_1115 (size=5958) 2024-11-20T10:31:30,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741936_1115 (size=5958) 2024-11-20T10:31:30,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741936_1115 (size=5958) 2024-11-20T10:31:30,307 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e61a42b0ace64281bd862b0c973ae9a7 2024-11-20T10:31:30,324 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/5fd8fda3cbdc4f369632cf626ec801a0 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732098690192/Put/seqid=0 2024-11-20T10:31:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741937_1116 (size=5958) 2024-11-20T10:31:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741937_1116 (size=5958) 2024-11-20T10:31:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741937_1116 (size=5958) 2024-11-20T10:31:30,330 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/5fd8fda3cbdc4f369632cf626ec801a0 2024-11-20T10:31:30,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e61a42b0ace64281bd862b0c973ae9a7 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/b/e61a42b0ace64281bd862b0c973ae9a7 2024-11-20T10:31:30,341 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/b/e61a42b0ace64281bd862b0c973ae9a7, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T10:31:30,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/5fd8fda3cbdc4f369632cf626ec801a0 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/c/5fd8fda3cbdc4f369632cf626ec801a0 2024-11-20T10:31:30,347 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/c/5fd8fda3cbdc4f369632cf626ec801a0, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T10:31:30,348 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for ff56393222adf494822dde15e6906700 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-11-20T10:31:30,348 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000035 2024-11-20T10:31:30,349 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,349 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,350 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:30,351 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ff56393222adf494822dde15e6906700 2024-11-20T10:31:30,353 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-20T10:31:30,354 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ff56393222adf494822dde15e6906700; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65067039, jitterRate=-0.03042556345462799}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:30,354 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ff56393222adf494822dde15e6906700: Writing region info on filesystem at 1732098690270Initializing all the Stores at 1732098690271 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690271Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690271Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098690271Obtaining lock to block concurrent updates at 1732098690287 (+16 ms)Preparing flush snapshotting stores in ff56393222adf494822dde15e6906700 at 1732098690287Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732098690287Flushing stores of testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. at 1732098690287Flushing ff56393222adf494822dde15e6906700/b: creating writer at 1732098690287Flushing ff56393222adf494822dde15e6906700/b: appending metadata at 1732098690300 (+13 ms)Flushing ff56393222adf494822dde15e6906700/b: closing flushed file at 1732098690300Flushing ff56393222adf494822dde15e6906700/c: creating writer at 1732098690311 (+11 ms)Flushing ff56393222adf494822dde15e6906700/c: appending metadata at 1732098690324 (+13 ms)Flushing ff56393222adf494822dde15e6906700/c: closing flushed file at 1732098690324Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fba66ff: reopening flushed file at 1732098690335 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ccbdef: reopening flushed file at 1732098690341 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for ff56393222adf494822dde15e6906700 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1732098690348 (+7 ms)Cleaning up temporary data from old regions at 1732098690349 (+1 ms)Region opened successfully at 1732098690354 (+5 ms) 2024-11-20T10:31:30,414 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, size=0 (0bytes) 2024-11-20T10:31:30,414 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 might be still open, length is 0 2024-11-20T10:31:30,414 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 2024-11-20T10:31:30,415 WARN [IPC Server handler 2 on default port 36923 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-20T10:31:30,415 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 after 1ms 2024-11-20T10:31:32,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:50822 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50822 dst: /127.0.0.1:38347 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38347 remote=/127.0.0.1:50822]. Total timeout mills is 60000, 57956 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:32,426 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:51214 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:46093:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51214 dst: /127.0.0.1:46093 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:32,426 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1714294407_22 at /127.0.0.1:53624 [Receiving block BP-1354523497-172.17.0.2-1732098649661:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53624 dst: /127.0.0.1:39533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:32,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741935_1117 (size=5215) 2024-11-20T10:31:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741935_1117 (size=5215) 2024-11-20T10:31:34,417 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 after 4003ms 2024-11-20T10:31:34,423 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 took 4009ms 2024-11-20T10:31:34,426 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245; continuing. 2024-11-20T10:31:34,426 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 so closing down 2024-11-20T10:31:34,426 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T10:31:34,428 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732098690245.temp 2024-11-20T10:31:34,430 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp 2024-11-20T10:31:34,430 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T10:31:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741938_1118 (size=5223) 2024-11-20T10:31:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741938_1118 (size=5223) 2024-11-20T10:31:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741938_1118 (size=5223) 2024-11-20T10:31:34,438 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T10:31:34,439 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 2024-11-20T10:31:34,439 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T10:31:34,439 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, journal: Splitting hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, size=0 (0bytes) at 1732098690414Finishing writing output for hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 so closing down at 1732098694426 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp at 1732098694430 (+4 ms)3 split writer threads finished at 1732098694430Closed recovered edits writer path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732098694438 (+8 ms)Rename recovered edits hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000037-wal.1732098690245.temp to hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 at 1732098694439 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245, size=0, length=0, corrupted=false, cancelled=false at 1732098694439 2024-11-20T10:31:34,440 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 to hdfs://localhost:36923/hbase/oldWALs/wal.1732098690245 2024-11-20T10:31:34,441 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 2024-11-20T10:31:34,441 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T10:31:34,443 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36923/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060, archiveDir=hdfs://localhost:36923/hbase/oldWALs, maxLogs=32 2024-11-20T10:31:34,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098694443, exclude list is [], retry=0 2024-11-20T10:31:34,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46093,DS-f1d3e970-fcd2-4a6e-9dea-eed5d2e19347,DISK] 2024-11-20T10:31:34,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39533,DS-598a40d1-0b51-4df7-942d-94be907a94aa,DISK] 2024-11-20T10:31:34,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38347,DS-54afc2a5-3649-437e-8ee9-0b42ca4bcbcc,DISK] 2024-11-20T10:31:34,460 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098694443 2024-11-20T10:31:34,460 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34509:34509),(127.0.0.1/127.0.0.1:34305:34305),(127.0.0.1/127.0.0.1:41233:41233)] 2024-11-20T10:31:34,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T10:31:34,462 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,462 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName a 2024-11-20T10:31:34,462 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:34,467 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/42a8ee940dba4261ab456a889bd73624 2024-11-20T10:31:34,467 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:34,467 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,468 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName b 2024-11-20T10:31:34,468 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:34,473 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/b/e61a42b0ace64281bd862b0c973ae9a7 2024-11-20T10:31:34,473 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:34,473 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,474 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff56393222adf494822dde15e6906700 columnFamilyName c 2024-11-20T10:31:34,474 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T10:31:34,478 DEBUG [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/c/5fd8fda3cbdc4f369632cf626ec801a0 2024-11-20T10:31:34,479 INFO [StoreOpener-ff56393222adf494822dde15e6906700-1 {}] regionserver.HStore(327): Store=ff56393222adf494822dde15e6906700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T10:31:34,479 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,481 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,482 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 2024-11-20T10:31:34,487 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 2024-11-20T10:31:34,487 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ff56393222adf494822dde15e6906700 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T10:31:34,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/404248fa875d40fc97bc1f88e433e0d5 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732098690360/Put/seqid=0 2024-11-20T10:31:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741940_1120 (size=5958) 2024-11-20T10:31:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741940_1120 (size=5958) 2024-11-20T10:31:34,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741940_1120 (size=5958) 2024-11-20T10:31:34,508 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/404248fa875d40fc97bc1f88e433e0d5 2024-11-20T10:31:34,530 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e8202fe16b004079b3f5903339d1c514 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732098690367/Put/seqid=0 2024-11-20T10:31:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741941_1121 (size=5958) 2024-11-20T10:31:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741941_1121 (size=5958) 2024-11-20T10:31:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741941_1121 (size=5958) 2024-11-20T10:31:34,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e8202fe16b004079b3f5903339d1c514 2024-11-20T10:31:34,555 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/7443bbc57e83462bb4c5d2fca72339a7 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732098690373/Put/seqid=0 2024-11-20T10:31:34,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741942_1122 (size=5958) 2024-11-20T10:31:34,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741942_1122 (size=5958) 2024-11-20T10:31:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741942_1122 (size=5958) 2024-11-20T10:31:34,562 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/7443bbc57e83462bb4c5d2fca72339a7 2024-11-20T10:31:34,567 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/a/404248fa875d40fc97bc1f88e433e0d5 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/404248fa875d40fc97bc1f88e433e0d5 2024-11-20T10:31:34,572 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/a/404248fa875d40fc97bc1f88e433e0d5, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T10:31:34,573 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/b/e8202fe16b004079b3f5903339d1c514 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/b/e8202fe16b004079b3f5903339d1c514 2024-11-20T10:31:34,578 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/b/e8202fe16b004079b3f5903339d1c514, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T10:31:34,579 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/.tmp/c/7443bbc57e83462bb4c5d2fca72339a7 as hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/c/7443bbc57e83462bb4c5d2fca72339a7 2024-11-20T10:31:34,584 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/c/7443bbc57e83462bb4c5d2fca72339a7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T10:31:34,584 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ff56393222adf494822dde15e6906700 in 97ms, sequenceid=66, compaction requested=false; wal=null 2024-11-20T10:31:34,585 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/0000000000000000066 2024-11-20T10:31:34,586 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,586 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,587 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T10:31:34,588 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ff56393222adf494822dde15e6906700 2024-11-20T10:31:34,590 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/hbase/data/default/testReplayEditsWrittenViaHRegion/ff56393222adf494822dde15e6906700/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-20T10:31:34,591 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ff56393222adf494822dde15e6906700; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68954735, jitterRate=0.027505621314048767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T10:31:34,592 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ff56393222adf494822dde15e6906700: Writing region info on filesystem at 1732098694461Initializing all the Stores at 1732098694461Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098694461Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098694462 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732098694462Obtaining lock to block concurrent updates at 1732098694487 (+25 ms)Preparing flush snapshotting stores in ff56393222adf494822dde15e6906700 at 1732098694487Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732098694487Flushing stores of testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. at 1732098694487Flushing ff56393222adf494822dde15e6906700/a: creating writer at 1732098694487Flushing ff56393222adf494822dde15e6906700/a: appending metadata at 1732098694501 (+14 ms)Flushing ff56393222adf494822dde15e6906700/a: closing flushed file at 1732098694501Flushing ff56393222adf494822dde15e6906700/b: creating writer at 1732098694513 (+12 ms)Flushing ff56393222adf494822dde15e6906700/b: appending metadata at 1732098694530 (+17 ms)Flushing ff56393222adf494822dde15e6906700/b: closing flushed file at 1732098694530Flushing ff56393222adf494822dde15e6906700/c: creating writer at 1732098694541 (+11 ms)Flushing ff56393222adf494822dde15e6906700/c: appending metadata at 1732098694555 (+14 ms)Flushing ff56393222adf494822dde15e6906700/c: closing flushed file at 1732098694555Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58eba74: reopening flushed file at 1732098694566 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16852d5d: reopening flushed file at 1732098694572 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bcd1e05: reopening flushed file at 1732098694578 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ff56393222adf494822dde15e6906700 in 97ms, sequenceid=66, compaction requested=false; wal=null at 1732098694584 (+6 ms)Cleaning up temporary data from old regions at 1732098694586 (+2 ms)Region opened successfully at 1732098694591 (+5 ms) 2024-11-20T10:31:34,603 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ff56393222adf494822dde15e6906700, disabling compactions & flushes 2024-11-20T10:31:34,603 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:34,603 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:34,603 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. after waiting 0 ms 2024-11-20T10:31:34,603 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:34,605 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732098690061.ff56393222adf494822dde15e6906700. 2024-11-20T10:31:34,605 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ff56393222adf494822dde15e6906700: Waiting for close lock at 1732098694603Disabling compacts and flushes for region at 1732098694603Disabling writes for close at 1732098694603Writing region close event to WAL at 1732098694605 (+2 ms)Closed at 1732098694605 2024-11-20T10:31:34,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741939_1119 (size=93) 2024-11-20T10:31:34,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741939_1119 (size=93) 2024-11-20T10:31:34,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741939_1119 (size=93) 2024-11-20T10:31:34,613 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T10:31:34,613 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732098694443) 2024-11-20T10:31:34,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 440) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904273720_22 at /127.0.0.1:43808 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904273720_22 at /127.0.0.1:41938 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904273720_22 at /127.0.0.1:53476 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:36923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36923 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 226), ProcessCount=11 (was 11), AvailableMemoryMB=8104 (was 8120) 2024-11-20T10:31:34,629 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-20T10:31:34,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T10:31:34,629 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:31:34,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:31:34,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T10:31:34,630 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T10:31:34,630 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1543425285, stopped=false 2024-11-20T10:31:34,631 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8c533afd331c,38325,1732098654332 2024-11-20T10:31:34,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-20T10:31:34,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:34,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-20T10:31:34,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:34,680 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-20T10:31:34,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-20T10:31:34,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T10:31:34,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T10:31:34,686 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T10:31:34,686 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:31:34,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:31:34,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:31:34,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T10:31:34,687 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T10:31:34,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:31:34,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,687 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8c533afd331c,40989,1732098655109' ***** 2024-11-20T10:31:34,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T10:31:34,687 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8c533afd331c,38107,1732098655202' ***** 2024-11-20T10:31:34,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T10:31:34,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:31:34,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:31:34,688 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T10:31:34,688 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(959): stopping server 8c533afd331c,38107,1732098655202 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8c533afd331c:38107. 2024-11-20T10:31:34,688 DEBUG [RS:1;8c533afd331c:38107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:31:34,688 DEBUG [RS:1;8c533afd331c:38107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T10:31:34,688 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T10:31:34,689 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T10:31:34,689 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T10:31:34,689 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T10:31:34,689 DEBUG [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(3091): Received CLOSE for 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:34,689 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(959): stopping server 8c533afd331c,40989,1732098655109 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8c533afd331c:40989. 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73ab4c4b40d6b3373ca57987a514eca4, disabling compactions & flushes 2024-11-20T10:31:34,689 DEBUG [RS:0;8c533afd331c:40989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T10:31:34,689 DEBUG [RS:0;8c533afd331c:40989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,689 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:34,689 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. after waiting 0 ms 2024-11-20T10:31:34,689 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1325): Online Regions={73ab4c4b40d6b3373ca57987a514eca4=testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4.} 2024-11-20T10:31:34,689 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-20T10:31:34,689 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:34,689 DEBUG [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1351): Waiting on 73ab4c4b40d6b3373ca57987a514eca4 2024-11-20T10:31:34,692 INFO [regionserver/8c533afd331c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:34,692 INFO [regionserver/8c533afd331c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:34,693 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/default/testReplayEditsAfterRegionMovedWithMultiCF/73ab4c4b40d6b3373ca57987a514eca4/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-20T10:31:34,694 INFO [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:34,694 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73ab4c4b40d6b3373ca57987a514eca4: Waiting for close lock at 1732098694689Running coprocessor pre-close hooks at 1732098694689Disabling compacts and flushes for region at 1732098694689Disabling writes for close at 1732098694689Writing region close event to WAL at 1732098694690 (+1 ms)Running coprocessor post-close hooks at 1732098694694 (+4 ms)Closed at 1732098694694 2024-11-20T10:31:34,694 DEBUG [RS_CLOSE_REGION-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4. 2024-11-20T10:31:34,694 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/03e2d40d6ebf4fb7be744d0beb2a83f5 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732098669544.73ab4c4b40d6b3373ca57987a514eca4./info:regioninfo/1732098673755/Put/seqid=0 2024-11-20T10:31:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741943_1123 (size=6778) 2024-11-20T10:31:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741943_1123 (size=6778) 2024-11-20T10:31:34,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741943_1123 (size=6778) 2024-11-20T10:31:34,703 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/03e2d40d6ebf4fb7be744d0beb2a83f5 2024-11-20T10:31:34,710 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/.tmp/info/03e2d40d6ebf4fb7be744d0beb2a83f5 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/info/03e2d40d6ebf4fb7be744d0beb2a83f5 2024-11-20T10:31:34,717 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/info/03e2d40d6ebf4fb7be744d0beb2a83f5, entries=8, sequenceid=23, filesize=6.6 K 2024-11-20T10:31:34,718 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false 2024-11-20T10:31:34,723 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-20T10:31:34,724 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T10:31:34,724 INFO [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T10:31:34,724 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732098694689Running coprocessor pre-close hooks at 1732098694689Disabling compacts and flushes for region at 1732098694689Disabling writes for close at 1732098694689Obtaining lock to block concurrent updates at 1732098694689Preparing flush snapshotting stores in 1588230740 at 1732098694689Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732098694690 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732098694690Flushing 1588230740/info: creating writer at 1732098694690Flushing 1588230740/info: appending metadata at 1732098694694 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732098694694Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@141df1e5: reopening flushed file at 1732098694709 (+15 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false at 1732098694718 (+9 ms)Writing region close event to WAL at 1732098694719 (+1 ms)Running coprocessor post-close hooks at 1732098694723 (+4 ms)Closed at 1732098694724 (+1 ms) 2024-11-20T10:31:34,724 DEBUG [RS_CLOSE_META-regionserver/8c533afd331c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T10:31:34,889 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(976): stopping server 8c533afd331c,38107,1732098655202; all regions closed. 2024-11-20T10:31:34,890 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(976): stopping server 8c533afd331c,40989,1732098655109; all regions closed. 2024-11-20T10:31:34,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741835_1011 (size=3433) 2024-11-20T10:31:34,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741835_1011 (size=3433) 2024-11-20T10:31:34,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741835_1011 (size=3433) 2024-11-20T10:31:34,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741893_1071 (size=2422) 2024-11-20T10:31:34,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741893_1071 (size=2422) 2024-11-20T10:31:34,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741893_1071 (size=2422) 2024-11-20T10:31:34,898 DEBUG [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs 2024-11-20T10:31:34,899 INFO [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8c533afd331c%2C38107%2C1732098655202.meta:.meta(num 1732098672832) 2024-11-20T10:31:34,899 DEBUG [RS:0;8c533afd331c:40989 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs 2024-11-20T10:31:34,899 INFO [RS:0;8c533afd331c:40989 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8c533afd331c%2C40989%2C1732098655109:(num 1732098656718) 2024-11-20T10:31:34,899 DEBUG [RS:0;8c533afd331c:40989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,899 INFO [RS:0;8c533afd331c:40989 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:34,899 INFO [RS:0;8c533afd331c:40989 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:31:34,899 INFO [RS:0;8c533afd331c:40989 {}] hbase.ChoreService(370): Chore service for: regionserver/8c533afd331c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T10:31:34,899 INFO [RS:0;8c533afd331c:40989 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T10:31:34,900 INFO [RS:0;8c533afd331c:40989 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T10:31:34,900 INFO [regionserver/8c533afd331c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:31:34,900 INFO [RS:0;8c533afd331c:40989 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T10:31:34,900 INFO [RS:0;8c533afd331c:40989 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:31:34,900 INFO [RS:0;8c533afd331c:40989 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40989 2024-11-20T10:31:34,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741834_1010 (size=93) 2024-11-20T10:31:34,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741834_1010 (size=93) 2024-11-20T10:31:34,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741834_1010 (size=93) 2024-11-20T10:31:34,904 DEBUG [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/oldWALs 2024-11-20T10:31:34,904 INFO [RS:1;8c533afd331c:38107 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8c533afd331c%2C38107%2C1732098655202:(num 1732098656718) 2024-11-20T10:31:34,904 DEBUG [RS:1;8c533afd331c:38107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T10:31:34,904 INFO [RS:1;8c533afd331c:38107 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T10:31:34,904 INFO [RS:1;8c533afd331c:38107 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:31:34,904 INFO [RS:1;8c533afd331c:38107 {}] hbase.ChoreService(370): Chore service for: regionserver/8c533afd331c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T10:31:34,905 INFO [RS:1;8c533afd331c:38107 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:31:34,905 INFO [regionserver/8c533afd331c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:31:34,905 INFO [RS:1;8c533afd331c:38107 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38107 2024-11-20T10:31:34,960 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8c533afd331c,40989,1732098655109 2024-11-20T10:31:34,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T10:31:34,960 INFO [RS:0;8c533afd331c:40989 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:31:34,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8c533afd331c,38107,1732098655202 2024-11-20T10:31:34,970 INFO [RS:1;8c533afd331c:38107 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:31:34,981 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8c533afd331c,40989,1732098655109] 2024-11-20T10:31:35,002 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8c533afd331c,40989,1732098655109 already deleted, retry=false 2024-11-20T10:31:35,002 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8c533afd331c,40989,1732098655109 expired; onlineServers=1 2024-11-20T10:31:35,002 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8c533afd331c,38107,1732098655202] 2024-11-20T10:31:35,012 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8c533afd331c,38107,1732098655202 already deleted, retry=false 2024-11-20T10:31:35,013 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8c533afd331c,38107,1732098655202 expired; onlineServers=0 2024-11-20T10:31:35,013 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8c533afd331c,38325,1732098654332' ***** 2024-11-20T10:31:35,013 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T10:31:35,013 INFO [M:0;8c533afd331c:38325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T10:31:35,014 INFO [M:0;8c533afd331c:38325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T10:31:35,014 DEBUG [M:0;8c533afd331c:38325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T10:31:35,014 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T10:31:35,014 DEBUG [M:0;8c533afd331c:38325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T10:31:35,014 DEBUG [master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098656358 {}] cleaner.HFileCleaner(306): Exit Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.large.0-1732098656358,5,FailOnTimeoutGroup] 2024-11-20T10:31:35,014 DEBUG [master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098656360 {}] cleaner.HFileCleaner(306): Exit Thread[master/8c533afd331c:0:becomeActiveMaster-HFileCleaner.small.0-1732098656360,5,FailOnTimeoutGroup] 2024-11-20T10:31:35,015 INFO [M:0;8c533afd331c:38325 {}] hbase.ChoreService(370): Chore service for: master/8c533afd331c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T10:31:35,015 INFO [M:0;8c533afd331c:38325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T10:31:35,015 DEBUG [M:0;8c533afd331c:38325 {}] master.HMaster(1795): Stopping service threads 2024-11-20T10:31:35,015 INFO [M:0;8c533afd331c:38325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T10:31:35,016 INFO [M:0;8c533afd331c:38325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T10:31:35,017 INFO [M:0;8c533afd331c:38325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T10:31:35,017 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T10:31:35,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T10:31:35,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T10:31:35,023 DEBUG [M:0;8c533afd331c:38325 {}] zookeeper.ZKUtil(347): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T10:31:35,023 WARN [M:0;8c533afd331c:38325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T10:31:35,024 INFO [M:0;8c533afd331c:38325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/.lastflushedseqids 2024-11-20T10:31:35,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741944_1124 (size=119) 2024-11-20T10:31:35,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741944_1124 (size=119) 2024-11-20T10:31:35,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741944_1124 (size=119) 2024-11-20T10:31:35,040 INFO [M:0;8c533afd331c:38325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T10:31:35,040 INFO [M:0;8c533afd331c:38325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T10:31:35,040 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T10:31:35,040 INFO [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:31:35,040 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:31:35,040 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T10:31:35,040 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:31:35,040 INFO [M:0;8c533afd331c:38325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-20T10:31:35,057 DEBUG [M:0;8c533afd331c:38325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e7691d07574e4f40b9d48347bef875f7 is 82, key is hbase:meta,,1/info:regioninfo/1732098673000/Put/seqid=0 2024-11-20T10:31:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741945_1125 (size=6063) 2024-11-20T10:31:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741945_1125 (size=6063) 2024-11-20T10:31:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741945_1125 (size=6063) 2024-11-20T10:31:35,064 INFO [M:0;8c533afd331c:38325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e7691d07574e4f40b9d48347bef875f7 2024-11-20T10:31:35,081 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,081 INFO [RS:0;8c533afd331c:40989 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:31:35,081 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40989-0x10157cbf93d0001, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,081 INFO [RS:0;8c533afd331c:40989 {}] regionserver.HRegionServer(1031): Exiting; stopping=8c533afd331c,40989,1732098655109; zookeeper connection closed. 2024-11-20T10:31:35,081 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e2b198 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e2b198 2024-11-20T10:31:35,083 DEBUG [M:0;8c533afd331c:38325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7cecf099bee41c38c5d3d27dbdf5830 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732098669970/Put/seqid=0 2024-11-20T10:31:35,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741946_1126 (size=7908) 2024-11-20T10:31:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741946_1126 (size=7908) 2024-11-20T10:31:35,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741946_1126 (size=7908) 2024-11-20T10:31:35,090 INFO [M:0;8c533afd331c:38325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7cecf099bee41c38c5d3d27dbdf5830 2024-11-20T10:31:35,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,091 INFO [RS:1;8c533afd331c:38107 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:31:35,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38107-0x10157cbf93d0002, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,091 INFO [RS:1;8c533afd331c:38107 {}] regionserver.HRegionServer(1031): Exiting; stopping=8c533afd331c,38107,1732098655202; zookeeper connection closed. 2024-11-20T10:31:35,092 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@62b156e4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@62b156e4 2024-11-20T10:31:35,092 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T10:31:35,095 INFO [M:0;8c533afd331c:38325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7cecf099bee41c38c5d3d27dbdf5830 2024-11-20T10:31:35,109 DEBUG [M:0;8c533afd331c:38325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2352e31fb43946059fb909ef614a764c is 69, key is 8c533afd331c,38107,1732098655202/rs:state/1732098656415/Put/seqid=0 2024-11-20T10:31:35,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741947_1127 (size=5440) 2024-11-20T10:31:35,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741947_1127 (size=5440) 2024-11-20T10:31:35,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741947_1127 (size=5440) 2024-11-20T10:31:35,115 INFO [M:0;8c533afd331c:38325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2352e31fb43946059fb909ef614a764c 2024-11-20T10:31:35,119 INFO [M:0;8c533afd331c:38325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2352e31fb43946059fb909ef614a764c 2024-11-20T10:31:35,120 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e7691d07574e4f40b9d48347bef875f7 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e7691d07574e4f40b9d48347bef875f7 2024-11-20T10:31:35,124 INFO [M:0;8c533afd331c:38325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e7691d07574e4f40b9d48347bef875f7, entries=14, sequenceid=207, filesize=5.9 K 2024-11-20T10:31:35,125 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7cecf099bee41c38c5d3d27dbdf5830 as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7cecf099bee41c38c5d3d27dbdf5830 2024-11-20T10:31:35,129 INFO [M:0;8c533afd331c:38325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7cecf099bee41c38c5d3d27dbdf5830 2024-11-20T10:31:35,129 INFO [M:0;8c533afd331c:38325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7cecf099bee41c38c5d3d27dbdf5830, entries=21, sequenceid=207, filesize=7.7 K 2024-11-20T10:31:35,130 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2352e31fb43946059fb909ef614a764c as hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2352e31fb43946059fb909ef614a764c 2024-11-20T10:31:35,134 INFO [M:0;8c533afd331c:38325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2352e31fb43946059fb909ef614a764c 2024-11-20T10:31:35,134 INFO [M:0;8c533afd331c:38325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36923/user/jenkins/test-data/4ec43e40-e8f0-455a-ccb4-60426330aa87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2352e31fb43946059fb909ef614a764c, entries=3, sequenceid=207, filesize=5.3 K 2024-11-20T10:31:35,135 INFO [M:0;8c533afd331c:38325 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false 2024-11-20T10:31:35,136 INFO [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T10:31:35,136 DEBUG [M:0;8c533afd331c:38325 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732098695040Disabling compacts and flushes for region at 1732098695040Disabling writes for close at 1732098695040Obtaining lock to block concurrent updates at 1732098695040Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732098695040Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1732098695040Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732098695041 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732098695041Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732098695057 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732098695057Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732098695067 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732098695083 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732098695083Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732098695095 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732098695108 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732098695108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f5d79c3: reopening flushed file at 1732098695119 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33fff931: reopening flushed file at 1732098695124 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e3ea322: reopening flushed file at 1732098695129 (+5 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false at 1732098695135 (+6 ms)Writing region close event to WAL at 1732098695136 (+1 ms)Closed at 1732098695136 2024-11-20T10:31:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741830_1006 (size=99448) 2024-11-20T10:31:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38347 is added to blk_1073741830_1006 (size=99448) 2024-11-20T10:31:35,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46093 is added to blk_1073741830_1006 (size=99448) 2024-11-20T10:31:35,140 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T10:31:35,140 INFO [M:0;8c533afd331c:38325 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T10:31:35,140 INFO [M:0;8c533afd331c:38325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38325 2024-11-20T10:31:35,140 INFO [M:0;8c533afd331c:38325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T10:31:35,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,249 INFO [M:0;8c533afd331c:38325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T10:31:35,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38325-0x10157cbf93d0000, quorum=127.0.0.1:63828, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T10:31:35,261 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098684672 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098684672 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732098675213/wal.1732098684672 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T10:31:35,264 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098675076 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098675076 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732098674307/wal.1732098675076 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T10:31:35,264 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732098665185/wal.1732098665280 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:35,269 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098685329 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098685329 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732098684871/wal.1732098685329 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T10:31:35,269 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732098685484/wal.1732098685564 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:35,270 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732098690060/wal.1732098690245 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:35,270 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732098658332/wal.1732098658401 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T10:31:35,272 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal.1732098658200 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal.1732098658200 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732098658034/wal.1732098658200 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T10:31:35,274 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal.1732098657796 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal.1732098657796 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732098657572/wal.1732098657796 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1714294407_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T10:31:35,279 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:31:35,281 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:31:35,281 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:31:35,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:31:35,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,STOPPED} 2024-11-20T10:31:35,284 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T10:31:35,284 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T10:31:35,284 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1354523497-172.17.0.2-1732098649661 (Datanode Uuid d1970de4-eae2-4112-a0d1-f0d815042ff4) service to localhost/127.0.0.1:36923 2024-11-20T10:31:35,284 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T10:31:35,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data5/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data6/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T10:31:35,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:31:35,292 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:31:35,292 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:31:35,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:31:35,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,STOPPED} 2024-11-20T10:31:35,293 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T10:31:35,293 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T10:31:35,293 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1354523497-172.17.0.2-1732098649661 (Datanode Uuid fb294b86-2cb6-45be-b146-b1a7befb00f0) service to localhost/127.0.0.1:36923 2024-11-20T10:31:35,293 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T10:31:35,294 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data3/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,294 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data4/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,294 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T10:31:35,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T10:31:35,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:31:35,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:31:35,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:31:35,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,STOPPED} 2024-11-20T10:31:35,299 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T10:31:35,299 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T10:31:35,299 WARN [BP-1354523497-172.17.0.2-1732098649661 heartbeating to localhost/127.0.0.1:36923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1354523497-172.17.0.2-1732098649661 (Datanode Uuid 056dc2d8-52c9-4108-b42e-d3d4b78ab6f2) service to localhost/127.0.0.1:36923 2024-11-20T10:31:35,299 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T10:31:35,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data1/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/cluster_f50f60fe-8559-e186-465f-b7520f491fc3/data/data2/current/BP-1354523497-172.17.0.2-1732098649661 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T10:31:35,300 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T10:31:35,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T10:31:35,305 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T10:31:35,305 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T10:31:35,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T10:31:35,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/9348be4c-355b-ec89-d60c-6c3b21d3e88d/hadoop.log.dir/,STOPPED} 2024-11-20T10:31:35,315 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T10:31:35,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down