2024-11-25 12:44:51,688 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-25 12:44:51,700 main DEBUG Took 0.009903 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 12:44:51,701 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 12:44:51,701 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 12:44:51,702 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 12:44:51,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,710 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 12:44:51,723 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,726 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,728 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,728 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,730 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,732 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,733 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,734 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,734 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:44:51,736 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,736 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 12:44:51,738 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:44:51,739 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 12:44:51,741 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 12:44:51,741 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 12:44:51,742 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 12:44:51,743 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 12:44:51,751 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 12:44:51,754 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 12:44:51,756 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 12:44:51,757 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 12:44:51,757 main DEBUG createAppenders(={Console}) 2024-11-25 12:44:51,759 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-25 12:44:51,759 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-25 12:44:51,760 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-25 12:44:51,760 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 12:44:51,761 main DEBUG OutputStream closed 2024-11-25 12:44:51,761 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 12:44:51,762 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 12:44:51,762 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-25 12:44:51,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 12:44:51,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 12:44:51,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 12:44:51,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 12:44:51,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 12:44:51,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 12:44:51,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 12:44:51,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 12:44:51,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 12:44:51,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 12:44:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 12:44:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 12:44:51,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 12:44:51,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 12:44:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 12:44:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 12:44:51,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 12:44:51,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 12:44:51,882 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 12:44:51,882 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-25 12:44:51,882 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 12:44:51,883 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-25T12:44:52,214 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6 2024-11-25 12:44:52,217 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 12:44:52,218 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T12:44:52,227 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-25T12:44:52,234 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-25T12:44:52,254 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T12:44:52,308 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T12:44:52,309 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T12:44:52,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T12:44:52,350 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718, deleteOnExit=true 2024-11-25T12:44:52,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T12:44:52,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/test.cache.data in system properties and HBase conf 2024-11-25T12:44:52,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T12:44:52,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir in system properties and HBase conf 2024-11-25T12:44:52,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T12:44:52,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T12:44:52,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T12:44:52,497 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T12:44:52,713 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T12:44:52,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:44:52,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:44:52,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T12:44:52,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:44:52,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T12:44:52,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T12:44:52,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:44:52,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:44:52,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T12:44:52,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/nfs.dump.dir in system properties and HBase conf 2024-11-25T12:44:52,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/java.io.tmpdir in system properties and HBase conf 2024-11-25T12:44:52,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:44:52,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T12:44:52,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T12:44:54,538 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T12:44:54,639 INFO [Time-limited test {}] log.Log(170): Logging initialized @3951ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T12:44:54,759 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:44:54,866 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:44:54,944 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:44:54,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:44:54,950 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:44:54,985 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:44:54,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:44:54,995 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:44:55,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/java.io.tmpdir/jetty-localhost-34933-hadoop-hdfs-3_4_1-tests_jar-_-any-16952413705949499481/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:44:55,266 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:34933} 2024-11-25T12:44:55,267 INFO [Time-limited test {}] server.Server(415): Started @4579ms 2024-11-25T12:44:55,972 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:44:55,984 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:44:55,987 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:44:55,987 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:44:55,988 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:44:55,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:44:55,992 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:44:56,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/java.io.tmpdir/jetty-localhost-38671-hadoop-hdfs-3_4_1-tests_jar-_-any-18255919063636251226/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:44:56,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:38671} 2024-11-25T12:44:56,137 INFO [Time-limited test {}] server.Server(415): Started @5449ms 2024-11-25T12:44:56,194 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:44:56,386 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:44:56,405 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:44:56,412 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:44:56,413 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:44:56,413 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:44:56,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:44:56,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:44:56,614 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/java.io.tmpdir/jetty-localhost-45175-hadoop-hdfs-3_4_1-tests_jar-_-any-3034067726955904574/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:44:56,618 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:45175} 2024-11-25T12:44:56,618 INFO [Time-limited test {}] server.Server(415): Started @5930ms 2024-11-25T12:44:56,627 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:44:56,681 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:44:56,687 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:44:56,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:44:56,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:44:56,691 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T12:44:56,693 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:44:56,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:44:56,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/java.io.tmpdir/jetty-localhost-45357-hadoop-hdfs-3_4_1-tests_jar-_-any-15317521129293120663/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:44:56,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:45357} 2024-11-25T12:44:56,814 INFO [Time-limited test {}] server.Server(415): Started @6126ms 2024-11-25T12:44:56,815 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:44:57,835 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data4/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,835 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data3/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,835 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data1/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,835 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data2/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,884 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:44:57,885 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:44:57,940 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x933bb6c2dfc032a with lease ID 0xbaaa5db5f4152feb: Processing first storage report for DS-772d4dd6-0bc9-494f-93a4-955af978e62c from datanode DatanodeRegistration(127.0.0.1:45631, datanodeUuid=8a0479a1-41ed-4b63-bc49-f178a653752d, infoPort=45865, infoSecurePort=0, ipcPort=36989, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x933bb6c2dfc032a with lease ID 0xbaaa5db5f4152feb: from storage DS-772d4dd6-0bc9-494f-93a4-955af978e62c node DatanodeRegistration(127.0.0.1:45631, datanodeUuid=8a0479a1-41ed-4b63-bc49-f178a653752d, infoPort=45865, infoSecurePort=0, ipcPort=36989, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T12:44:57,941 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data5/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3c514578da42cadb with lease ID 0xbaaa5db5f4152fec: Processing first storage report for DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f from datanode DatanodeRegistration(127.0.0.1:35695, datanodeUuid=2d68da22-7be8-4f1a-9926-dc2b6466d4c2, infoPort=44527, infoSecurePort=0, ipcPort=34915, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c514578da42cadb with lease ID 0xbaaa5db5f4152fec: from storage DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f node DatanodeRegistration(127.0.0.1:35695, datanodeUuid=2d68da22-7be8-4f1a-9926-dc2b6466d4c2, infoPort=44527, infoSecurePort=0, ipcPort=34915, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:44:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x933bb6c2dfc032a with lease ID 0xbaaa5db5f4152feb: Processing first storage report for DS-22e0176d-4f3c-481f-b2b1-eb96007565ce from datanode DatanodeRegistration(127.0.0.1:45631, datanodeUuid=8a0479a1-41ed-4b63-bc49-f178a653752d, infoPort=45865, infoSecurePort=0, ipcPort=36989, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x933bb6c2dfc032a with lease ID 0xbaaa5db5f4152feb: from storage DS-22e0176d-4f3c-481f-b2b1-eb96007565ce node DatanodeRegistration(127.0.0.1:45631, datanodeUuid=8a0479a1-41ed-4b63-bc49-f178a653752d, infoPort=45865, infoSecurePort=0, ipcPort=36989, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T12:44:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3c514578da42cadb with lease ID 0xbaaa5db5f4152fec: Processing first storage report for DS-6a87e5c1-dfc8-4a31-b4b5-2f49dcaaa8b8 from datanode DatanodeRegistration(127.0.0.1:35695, datanodeUuid=2d68da22-7be8-4f1a-9926-dc2b6466d4c2, infoPort=44527, infoSecurePort=0, ipcPort=34915, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c514578da42cadb with lease ID 0xbaaa5db5f4152fec: from storage DS-6a87e5c1-dfc8-4a31-b4b5-2f49dcaaa8b8 node DatanodeRegistration(127.0.0.1:35695, datanodeUuid=2d68da22-7be8-4f1a-9926-dc2b6466d4c2, infoPort=44527, infoSecurePort=0, ipcPort=34915, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:44:57,944 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data6/current/BP-322817772-172.17.0.3-1732538693888/current, will proceed with Du for space computation calculation, 2024-11-25T12:44:57,975 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:44:57,982 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e838540f1014895 with lease ID 0xbaaa5db5f4152fed: Processing first storage report for DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1 from datanode DatanodeRegistration(127.0.0.1:38775, datanodeUuid=135a58b5-b6c0-4766-9725-2aa3aa805de0, infoPort=44585, infoSecurePort=0, ipcPort=37429, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,983 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e838540f1014895 with lease ID 0xbaaa5db5f4152fed: from storage DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1 node DatanodeRegistration(127.0.0.1:38775, datanodeUuid=135a58b5-b6c0-4766-9725-2aa3aa805de0, infoPort=44585, infoSecurePort=0, ipcPort=37429, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:44:57,983 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e838540f1014895 with lease ID 0xbaaa5db5f4152fed: Processing first storage report for DS-b536457f-94d1-4efa-a0fd-a406cead9071 from datanode DatanodeRegistration(127.0.0.1:38775, datanodeUuid=135a58b5-b6c0-4766-9725-2aa3aa805de0, infoPort=44585, infoSecurePort=0, ipcPort=37429, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888) 2024-11-25T12:44:57,983 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e838540f1014895 with lease ID 0xbaaa5db5f4152fed: from storage DS-b536457f-94d1-4efa-a0fd-a406cead9071 node DatanodeRegistration(127.0.0.1:38775, datanodeUuid=135a58b5-b6c0-4766-9725-2aa3aa805de0, infoPort=44585, infoSecurePort=0, ipcPort=37429, storageInfo=lv=-57;cid=testClusterID;nsid=1673334621;c=1732538693888), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:44:58,035 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6 2024-11-25T12:44:58,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/zookeeper_0, clientPort=59707, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T12:44:58,162 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59707 2024-11-25T12:44:58,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:44:58,195 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:44:58,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:44:58,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:44:58,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:44:58,910 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 with version=8 2024-11-25T12:44:58,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/hbase-staging 2024-11-25T12:44:59,291 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:44:59,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:44:59,304 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:44:59,310 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:44:59,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:44:59,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:44:59,515 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T12:44:59,596 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T12:44:59,607 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T12:44:59,610 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:44:59,642 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33254 (auto-detected) 2024-11-25T12:44:59,643 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T12:44:59,670 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43183 2024-11-25T12:44:59,704 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43183 connecting to ZooKeeper ensemble=127.0.0.1:59707 2024-11-25T12:44:59,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431830x0, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:44:59,832 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43183-0x10173503b590000 connected 2024-11-25T12:44:59,990 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:44:59,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:00,014 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3, hbase.cluster.distributed=false 2024-11-25T12:45:00,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:00,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43183 2024-11-25T12:45:00,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43183 2024-11-25T12:45:00,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43183 2024-11-25T12:45:00,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43183 2024-11-25T12:45:00,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43183 2024-11-25T12:45:00,185 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:00,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,187 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:00,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,187 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:00,190 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:00,192 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:00,193 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35011 2024-11-25T12:45:00,196 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35011 connecting to ZooKeeper ensemble=127.0.0.1:59707 2024-11-25T12:45:00,198 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,202 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350110x0, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:00,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35011-0x10173503b590001 connected 2024-11-25T12:45:00,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:00,227 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:00,239 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:00,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:00,250 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:00,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35011 2024-11-25T12:45:00,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35011 2024-11-25T12:45:00,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35011 2024-11-25T12:45:00,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35011 2024-11-25T12:45:00,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35011 2024-11-25T12:45:00,283 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:00,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,285 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:00,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,285 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:00,285 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:00,286 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:00,288 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37815 2024-11-25T12:45:00,290 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37815 connecting to ZooKeeper ensemble=127.0.0.1:59707 2024-11-25T12:45:00,292 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,295 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,314 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378150x0, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:00,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37815-0x10173503b590002 connected 2024-11-25T12:45:00,318 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:00,319 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:00,326 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:00,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:00,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:00,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37815 2024-11-25T12:45:00,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37815 2024-11-25T12:45:00,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37815 2024-11-25T12:45:00,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37815 2024-11-25T12:45:00,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37815 2024-11-25T12:45:00,373 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ff8009e463e:0 server-side Connection retries=45 2024-11-25T12:45:00,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,374 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:45:00,374 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:45:00,374 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:45:00,374 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:45:00,375 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:45:00,376 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44737 2024-11-25T12:45:00,379 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44737 connecting to ZooKeeper ensemble=127.0.0.1:59707 2024-11-25T12:45:00,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447370x0, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:45:00,417 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44737-0x10173503b590003 connected 2024-11-25T12:45:00,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:00,418 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:45:00,425 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:45:00,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:45:00,434 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:45:00,441 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-25T12:45:00,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44737 2024-11-25T12:45:00,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44737 2024-11-25T12:45:00,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-25T12:45:00,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-25T12:45:00,475 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ff8009e463e:43183 2024-11-25T12:45:00,478 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ff8009e463e,43183,1732538699066 2024-11-25T12:45:00,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,493 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,495 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ff8009e463e,43183,1732538699066 2024-11-25T12:45:00,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:00,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,534 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:00,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:00,535 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,536 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T12:45:00,538 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ff8009e463e,43183,1732538699066 from backup master directory 2024-11-25T12:45:00,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,551 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ff8009e463e,43183,1732538699066 2024-11-25T12:45:00,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:45:00,553 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:00,554 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ff8009e463e,43183,1732538699066 2024-11-25T12:45:00,556 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T12:45:00,559 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T12:45:00,632 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/hbase.id] with ID: 0c41da9c-186e-443b-b8c3-e118f36dec4e 2024-11-25T12:45:00,633 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/.tmp/hbase.id 2024-11-25T12:45:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:45:00,669 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/.tmp/hbase.id]:[hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/hbase.id] 2024-11-25T12:45:00,764 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:45:00,772 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T12:45:00,803 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 28ms. 2024-11-25T12:45:00,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,871 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:00,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:45:00,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:45:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:45:00,956 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:45:00,958 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T12:45:00,965 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:01,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:45:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:45:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:45:01,027 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store 2024-11-25T12:45:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:45:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:45:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:45:01,067 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T12:45:01,072 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:01,074 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:45:01,074 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:01,075 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:01,077 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:45:01,077 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:01,078 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:01,080 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732538701074Disabling compacts and flushes for region at 1732538701074Disabling writes for close at 1732538701077 (+3 ms)Writing region close event to WAL at 1732538701078 (+1 ms)Closed at 1732538701078 2024-11-25T12:45:01,083 WARN [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/.initializing 2024-11-25T12:45:01,083 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/WALs/1ff8009e463e,43183,1732538699066 2024-11-25T12:45:01,092 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:01,111 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C43183%2C1732538699066, suffix=, logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/WALs/1ff8009e463e,43183,1732538699066, archiveDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/oldWALs, maxLogs=10 2024-11-25T12:45:01,144 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/WALs/1ff8009e463e,43183,1732538699066/1ff8009e463e%2C43183%2C1732538699066.1732538701115, exclude list is [], retry=0 2024-11-25T12:45:01,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:01,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:01,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:01,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:01,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T12:45:01,218 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-25T12:45:01,308 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/WALs/1ff8009e463e,43183,1732538699066/1ff8009e463e%2C43183%2C1732538699066.1732538701115 2024-11-25T12:45:01,312 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:01,313 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:01,314 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:01,318 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,319 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T12:45:01,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:01,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:01,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T12:45:01,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:01,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:01,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T12:45:01,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:01,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:01,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T12:45:01,442 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:01,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:01,445 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,451 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,453 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,460 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,461 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,471 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:45:01,480 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:45:01,487 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:01,488 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74824070, jitterRate=0.1149655282497406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:45:01,496 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732538701341Initializing all the Stores at 1732538701344 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538701345 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538701346 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538701346Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538701346Cleaning up temporary data from old regions at 1732538701461 (+115 ms)Region opened successfully at 1732538701495 (+34 ms) 2024-11-25T12:45:01,499 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T12:45:01,543 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab29bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:01,574 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T12:45:01,589 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T12:45:01,589 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T12:45:01,592 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T12:45:01,593 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T12:45:01,598 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T12:45:01,599 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T12:45:01,636 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T12:45:01,648 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T12:45:01,688 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T12:45:01,692 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T12:45:01,694 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T12:45:01,700 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T12:45:01,704 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T12:45:01,710 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T12:45:01,717 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T12:45:01,719 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T12:45:01,730 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T12:45:01,752 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T12:45:01,758 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T12:45:01,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:01,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:01,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:01,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,773 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:01,773 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,781 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ff8009e463e,43183,1732538699066, sessionid=0x10173503b590000, setting cluster-up flag (Was=false) 2024-11-25T12:45:01,813 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,842 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T12:45:01,844 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,43183,1732538699066 2024-11-25T12:45:01,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,867 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:01,897 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T12:45:01,899 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ff8009e463e,43183,1732538699066 2024-11-25T12:45:01,907 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T12:45:01,961 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(746): ClusterId : 0c41da9c-186e-443b-b8c3-e118f36dec4e 2024-11-25T12:45:01,964 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:01,967 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(746): ClusterId : 0c41da9c-186e-443b-b8c3-e118f36dec4e 2024-11-25T12:45:01,967 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:01,983 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(746): ClusterId : 0c41da9c-186e-443b-b8c3-e118f36dec4e 2024-11-25T12:45:01,983 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:45:01,998 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:01,998 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:01,999 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:01,999 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:02,004 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:45:02,004 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:45:02,018 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:02,018 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:02,018 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:45:02,019 DEBUG [RS:2;1ff8009e463e:44737 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fe902c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:02,019 DEBUG [RS:0;1ff8009e463e:35011 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d48fdf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:02,019 DEBUG [RS:1;1ff8009e463e:37815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c230177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ff8009e463e/172.17.0.3:0 2024-11-25T12:45:02,022 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:02,034 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T12:45:02,037 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ff8009e463e:35011 2024-11-25T12:45:02,038 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1ff8009e463e:44737 2024-11-25T12:45:02,039 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1ff8009e463e:37815 2024-11-25T12:45:02,040 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T12:45:02,041 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:02,041 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:02,041 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:45:02,041 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:02,041 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:02,041 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:45:02,041 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:02,041 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:02,041 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:45:02,044 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,43183,1732538699066 with port=35011, startcode=1732538700146 2024-11-25T12:45:02,044 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,43183,1732538699066 with port=37815, startcode=1732538700283 2024-11-25T12:45:02,044 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,43183,1732538699066 with port=44737, startcode=1732538700372 2024-11-25T12:45:02,048 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ff8009e463e,43183,1732538699066 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T12:45:02,056 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:02,056 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:02,056 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:02,057 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ff8009e463e:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:45:02,057 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ff8009e463e:0, corePoolSize=10, maxPoolSize=10 2024-11-25T12:45:02,057 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,057 DEBUG [RS:1;1ff8009e463e:37815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:02,057 DEBUG [RS:2;1ff8009e463e:44737 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:02,057 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:02,058 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,057 DEBUG [RS:0;1ff8009e463e:35011 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:45:02,070 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732538732069 2024-11-25T12:45:02,072 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T12:45:02,073 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T12:45:02,076 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T12:45:02,077 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T12:45:02,077 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T12:45:02,078 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T12:45:02,083 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,091 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T12:45:02,092 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T12:45:02,093 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T12:45:02,094 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:02,095 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T12:45:02,100 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T12:45:02,101 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T12:45:02,101 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:02,101 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:02,102 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54049, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:45:02,103 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538702102,5,FailOnTimeoutGroup] 2024-11-25T12:45:02,103 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:02,104 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T12:45:02,107 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538702103,5,FailOnTimeoutGroup] 2024-11-25T12:45:02,107 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,108 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T12:45:02,109 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,109 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,109 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T12:45:02,116 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,132 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 2024-11-25T12:45:02,133 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41185 2024-11-25T12:45:02,133 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:02,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:02,143 DEBUG [RS:2;1ff8009e463e:44737 {}] zookeeper.ZKUtil(111): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,143 WARN [RS:2;1ff8009e463e:44737 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:02,143 INFO [RS:2;1ff8009e463e:44737 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:02,144 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,149 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,44737,1732538700372] 2024-11-25T12:45:02,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,182 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 2024-11-25T12:45:02,182 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41185 2024-11-25T12:45:02,182 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:02,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:45:02,192 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T12:45:02,192 WARN [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T12:45:02,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:45:02,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:45:02,195 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T12:45:02,195 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 2024-11-25T12:45:02,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:45:02,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:45:02,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:45:02,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:02,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:02,224 DEBUG [RS:1;1ff8009e463e:37815 {}] zookeeper.ZKUtil(111): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,224 WARN [RS:1;1ff8009e463e:37815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:02,224 INFO [RS:1;1ff8009e463e:37815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:02,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,37815,1732538700283] 2024-11-25T12:45:02,224 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:45:02,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:45:02,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:02,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:02,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:45:02,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:45:02,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:02,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:02,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:45:02,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:45:02,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:02,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:02,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:45:02,251 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:02,251 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:02,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:45:02,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:02,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:02,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:45:02,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740 2024-11-25T12:45:02,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740 2024-11-25T12:45:02,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:45:02,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:45:02,263 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:45:02,267 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:45:02,272 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:02,274 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61105515, jitterRate=-0.08945687115192413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:45:02,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732538702221Initializing all the Stores at 1732538702226 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538702226Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538702227 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538702227Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538702227Cleaning up temporary data from old regions at 1732538702262 (+35 ms)Region opened successfully at 1732538702276 (+14 ms) 2024-11-25T12:45:02,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:45:02,276 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:45:02,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:45:02,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:45:02,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:45:02,281 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:02,281 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:02,281 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:45:02,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732538702276Disabling compacts and flushes for region at 1732538702276Disabling writes for close at 1732538702276Writing region close event to WAL at 1732538702280 (+4 ms)Closed at 1732538702281 (+1 ms) 2024-11-25T12:45:02,285 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:02,285 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T12:45:02,287 INFO [RS:1;1ff8009e463e:37815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:02,287 INFO [RS:2;1ff8009e463e:44737 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:02,287 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,287 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T12:45:02,293 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:02,293 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:02,293 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ff8009e463e,43183,1732538699066 with port=35011, startcode=1732538700146 2024-11-25T12:45:02,295 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,295 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] master.ServerManager(517): Registering regionserver=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,299 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 2024-11-25T12:45:02,299 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41185 2024-11-25T12:45:02,300 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:45:02,300 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:02,301 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:02,301 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,301 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,302 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,302 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,302 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,302 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:02,303 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,303 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,304 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,304 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,305 DEBUG [RS:2;1ff8009e463e:44737 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,305 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:45:02,305 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,305 DEBUG [RS:1;1ff8009e463e:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,307 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,307 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,308 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,308 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,308 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,308 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,44737,1732538700372-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:02,308 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T12:45:02,315 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,316 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,316 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:02,317 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,317 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,317 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,37815,1732538700283-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:02,318 DEBUG [RS:0;1ff8009e463e:35011 {}] zookeeper.ZKUtil(111): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,318 WARN [RS:0;1ff8009e463e:35011 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:45:02,318 INFO [RS:0;1ff8009e463e:35011 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:02,318 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ff8009e463e,35011,1732538700146] 2024-11-25T12:45:02,319 DEBUG [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,344 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:02,346 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,44737,1732538700372-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,347 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,347 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.Replication(171): 1ff8009e463e,44737,1732538700372 started 2024-11-25T12:45:02,351 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:02,351 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,37815,1732538700283-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,352 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,352 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.Replication(171): 1ff8009e463e,37815,1732538700283 started 2024-11-25T12:45:02,357 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:45:02,363 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:45:02,365 INFO [RS:0;1ff8009e463e:35011 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:02,365 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,366 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:45:02,368 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:45:02,372 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,372 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,44737,1732538700372, RpcServer on 1ff8009e463e/172.17.0.3:44737, sessionid=0x10173503b590003 2024-11-25T12:45:02,373 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:02,373 DEBUG [RS:2;1ff8009e463e:44737 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,374 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,44737,1732538700372' 2024-11-25T12:45:02,374 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:02,375 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:02,368 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,376 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,376 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,376 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:02,376 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,376 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:02,376 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,376 DEBUG [RS:2;1ff8009e463e:44737 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,376 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,37815,1732538700283, RpcServer on 1ff8009e463e/172.17.0.3:37815, sessionid=0x10173503b590002 2024-11-25T12:45:02,376 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,44737,1732538700372' 2024-11-25T12:45:02,376 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:02,376 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:1;1ff8009e463e:37815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,377 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,37815,1732538700283' 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:45:02,377 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,377 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:02,377 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:02,377 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,378 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ff8009e463e:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:45:02,378 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,378 DEBUG [RS:0;1ff8009e463e:35011 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ff8009e463e:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:45:02,378 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:02,378 DEBUG [RS:2;1ff8009e463e:44737 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:02,378 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:02,378 DEBUG [RS:1;1ff8009e463e:37815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,37815,1732538700283 2024-11-25T12:45:02,378 INFO [RS:2;1ff8009e463e:44737 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:02,378 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,37815,1732538700283' 2024-11-25T12:45:02,378 INFO [RS:2;1ff8009e463e:44737 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:02,378 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:02,379 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:02,380 DEBUG [RS:1;1ff8009e463e:37815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:02,380 INFO [RS:1;1ff8009e463e:37815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:02,380 INFO [RS:1;1ff8009e463e:37815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:02,386 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,387 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,387 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,387 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,387 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,387 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35011,1732538700146-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:02,411 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:45:02,411 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,35011,1732538700146-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,412 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,412 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.Replication(171): 1ff8009e463e,35011,1732538700146 started 2024-11-25T12:45:02,434 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:02,434 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1482): Serving as 1ff8009e463e,35011,1732538700146, RpcServer on 1ff8009e463e/172.17.0.3:35011, sessionid=0x10173503b590001 2024-11-25T12:45:02,435 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:45:02,435 DEBUG [RS:0;1ff8009e463e:35011 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,435 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35011,1732538700146' 2024-11-25T12:45:02,435 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:45:02,440 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:45:02,442 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:45:02,442 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:45:02,442 DEBUG [RS:0;1ff8009e463e:35011 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ff8009e463e,35011,1732538700146 2024-11-25T12:45:02,442 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ff8009e463e,35011,1732538700146' 2024-11-25T12:45:02,442 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:45:02,443 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:45:02,444 DEBUG [RS:0;1ff8009e463e:35011 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:45:02,445 INFO [RS:0;1ff8009e463e:35011 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:45:02,446 INFO [RS:0;1ff8009e463e:35011 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:45:02,460 WARN [1ff8009e463e:43183 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T12:45:02,485 INFO [RS:1;1ff8009e463e:37815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:02,486 INFO [RS:2;1ff8009e463e:44737 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:02,492 INFO [RS:1;1ff8009e463e:37815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C37815%2C1732538700283, suffix=, logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283, archiveDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs, maxLogs=32 2024-11-25T12:45:02,492 INFO [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C44737%2C1732538700372, suffix=, logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372, archiveDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs, maxLogs=32 2024-11-25T12:45:02,512 DEBUG [RS:1;1ff8009e463e:37815 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283/1ff8009e463e%2C37815%2C1732538700283.1732538702495, exclude list is [], retry=0 2024-11-25T12:45:02,514 DEBUG [RS:2;1ff8009e463e:44737 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372/1ff8009e463e%2C44737%2C1732538700372.1732538702496, exclude list is [], retry=0 2024-11-25T12:45:02,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:02,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:02,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:02,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:02,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:02,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:02,547 INFO [RS:0;1ff8009e463e:35011 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:02,561 INFO [RS:0;1ff8009e463e:35011 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C35011%2C1732538700146, suffix=, logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,35011,1732538700146, archiveDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs, maxLogs=32 2024-11-25T12:45:02,583 DEBUG [RS:0;1ff8009e463e:35011 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,35011,1732538700146/1ff8009e463e%2C35011%2C1732538700146.1732538702564, exclude list is [], retry=0 2024-11-25T12:45:02,603 INFO [RS:1;1ff8009e463e:37815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283/1ff8009e463e%2C37815%2C1732538700283.1732538702495 2024-11-25T12:45:02,605 INFO [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372/1ff8009e463e%2C44737%2C1732538700372.1732538702496 2024-11-25T12:45:02,606 DEBUG [RS:1;1ff8009e463e:37815 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:02,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:02,609 DEBUG [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:02,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:02,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:02,651 INFO [RS:0;1ff8009e463e:35011 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,35011,1732538700146/1ff8009e463e%2C35011%2C1732538700146.1732538702564 2024-11-25T12:45:02,651 DEBUG [RS:0;1ff8009e463e:35011 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:02,721 DEBUG [1ff8009e463e:43183 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-25T12:45:02,730 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:45:02,740 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:45:02,740 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:45:02,740 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T12:45:02,740 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:45:02,740 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:45:02,741 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T12:45:02,741 INFO [1ff8009e463e:43183 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:45:02,741 INFO [1ff8009e463e:43183 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:45:02,741 INFO [1ff8009e463e:43183 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T12:45:02,741 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:45:02,750 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:02,757 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,44737,1732538700372, state=OPENING 2024-11-25T12:45:02,797 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T12:45:02,808 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:02,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:02,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:02,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:02,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:02,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:02,810 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:02,810 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:02,811 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:45:02,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,44737,1732538700372}] 2024-11-25T12:45:03,004 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:45:03,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43233, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:45:03,025 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T12:45:03,026 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:03,027 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T12:45:03,031 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ff8009e463e%2C44737%2C1732538700372.meta, suffix=.meta, logDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372, archiveDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs, maxLogs=32 2024-11-25T12:45:03,052 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372/1ff8009e463e%2C44737%2C1732538700372.meta.1732538703034.meta, exclude list is [], retry=0 2024-11-25T12:45:03,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:03,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:03,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:03,093 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,44737,1732538700372/1ff8009e463e%2C44737%2C1732538700372.meta.1732538703034.meta 2024-11-25T12:45:03,093 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:03,093 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:03,095 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T12:45:03,097 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T12:45:03,103 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T12:45:03,109 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T12:45:03,110 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:03,110 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T12:45:03,111 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T12:45:03,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:45:03,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:45:03,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:03,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:03,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:45:03,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:45:03,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:03,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:03,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:45:03,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:45:03,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:03,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:03,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:45:03,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:45:03,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:03,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:45:03,130 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:45:03,132 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740 2024-11-25T12:45:03,139 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740 2024-11-25T12:45:03,142 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:45:03,142 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:45:03,144 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:45:03,147 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:45:03,148 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71640379, jitterRate=0.06752483546733856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:45:03,149 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T12:45:03,150 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732538703111Writing region info on filesystem at 1732538703112 (+1 ms)Initializing all the Stores at 1732538703114 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538703114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538703114Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538703114Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732538703114Cleaning up temporary data from old regions at 1732538703143 (+29 ms)Running coprocessor post-open hooks at 1732538703149 (+6 ms)Region opened successfully at 1732538703150 (+1 ms) 2024-11-25T12:45:03,158 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732538702992 2024-11-25T12:45:03,173 DEBUG [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T12:45:03,174 INFO [RS_OPEN_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T12:45:03,175 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:03,178 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ff8009e463e,44737,1732538700372, state=OPEN 2024-11-25T12:45:03,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:45:03,188 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:45:03,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:45:03,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:45:03,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:03,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:03,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:03,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:45:03,189 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:03,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T12:45:03,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ff8009e463e,44737,1732538700372 in 375 msec 2024-11-25T12:45:03,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T12:45:03,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 907 msec 2024-11-25T12:45:03,208 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:45:03,209 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T12:45:03,231 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:45:03,233 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,44737,1732538700372, seqNum=-1] 2024-11-25T12:45:03,258 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:45:03,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:45:03,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3290 sec 2024-11-25T12:45:03,292 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732538703292, completionTime=-1 2024-11-25T12:45:03,296 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-25T12:45:03,296 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T12:45:03,354 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-25T12:45:03,354 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732538763354 2024-11-25T12:45:03,355 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732538823354 2024-11-25T12:45:03,355 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 58 msec 2024-11-25T12:45:03,357 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T12:45:03,372 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,373 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,373 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,375 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ff8009e463e:43183, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,375 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,379 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,386 DEBUG [master/1ff8009e463e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T12:45:03,415 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.861sec 2024-11-25T12:45:03,417 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T12:45:03,418 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T12:45:03,419 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T12:45:03,420 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T12:45:03,420 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T12:45:03,421 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:45:03,422 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T12:45:03,427 DEBUG [master/1ff8009e463e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T12:45:03,428 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T12:45:03,429 INFO [master/1ff8009e463e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ff8009e463e,43183,1732538699066-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:03,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6adc31d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:45:03,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ff8009e463e,43183,-1 for getting cluster id 2024-11-25T12:45:03,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:45:03,519 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0c41da9c-186e-443b-b8c3-e118f36dec4e' 2024-11-25T12:45:03,522 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:45:03,523 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0c41da9c-186e-443b-b8c3-e118f36dec4e" 2024-11-25T12:45:03,523 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f41a244, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:45:03,524 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ff8009e463e,43183,-1] 2024-11-25T12:45:03,527 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:45:03,531 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:03,545 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:45:03,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a9e95a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:45:03,550 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:45:03,564 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ff8009e463e,44737,1732538700372, seqNum=-1] 2024-11-25T12:45:03,565 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:45:03,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:45:03,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ff8009e463e,43183,1732538699066 2024-11-25T12:45:03,605 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41185/hbase 2024-11-25T12:45:03,643 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=367, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=2641 2024-11-25T12:45:03,676 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:03,681 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:03,683 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:03,688 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13409718, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-13409718, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:03,708 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13409718/hregion-13409718.1732538703690, exclude list is [], retry=0 2024-11-25T12:45:03,724 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:03,735 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:03,736 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:03,791 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13409718/hregion-13409718.1732538703690 2024-11-25T12:45:03,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:03,796 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3e4249dccf225607b1521778c17a5c90, NAME => 'testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:03,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:45:03,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:45:03,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741838_1014 (size=64) 2024-11-25T12:45:03,853 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:03,857 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,860 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e4249dccf225607b1521778c17a5c90 columnFamilyName a 2024-11-25T12:45:03,860 DEBUG [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:03,861 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] regionserver.HStore(327): Store=3e4249dccf225607b1521778c17a5c90/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:03,861 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,864 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,864 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,867 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:03,876 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:03,877 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3e4249dccf225607b1521778c17a5c90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67032474, jitterRate=-0.0011382997035980225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3e4249dccf225607b1521778c17a5c90: Writing region info on filesystem at 1732538703854Initializing all the Stores at 1732538703856 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538703856Cleaning up temporary data from old regions at 1732538703864 (+8 ms)Region opened successfully at 1732538703879 (+15 ms) 2024-11-25T12:45:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3e4249dccf225607b1521778c17a5c90, disabling compactions & flushes 2024-11-25T12:45:03,879 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. 2024-11-25T12:45:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. 2024-11-25T12:45:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. after waiting 0 ms 2024-11-25T12:45:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. 2024-11-25T12:45:03,884 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. 2024-11-25T12:45:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3e4249dccf225607b1521778c17a5c90: Waiting for close lock at 1732538703879Disabling compacts and flushes for region at 1732538703879Disabling writes for close at 1732538703879Writing region close event to WAL at 1732538703883 (+4 ms)Closed at 1732538703884 (+1 ms) 2024-11-25T12:45:03,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741837_1013 (size=175) 2024-11-25T12:45:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741837_1013 (size=175) 2024-11-25T12:45:03,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741837_1013 (size=175) 2024-11-25T12:45:03,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:03,900 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13409718:(num 1732538703690) 2024-11-25T12:45:03,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:45:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741839_1015 (size=469) 2024-11-25T12:45:04,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741839_1015 (size=469) 2024-11-25T12:45:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741839_1015 (size=469) 2024-11-25T12:45:04,005 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:45:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741840_1016 (size=322) 2024-11-25T12:45:04,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741840_1016 (size=322) 2024-11-25T12:45:04,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741840_1016 (size=322) 2024-11-25T12:45:04,127 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1, size=469 (469bytes) 2024-11-25T12:45:04,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-25T12:45:04,128 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-25T12:45:04,128 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1 2024-11-25T12:45:04,134 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1 after 4ms 2024-11-25T12:45:04,171 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:04,172 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1 took 47ms 2024-11-25T12:45:04,179 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1 so closing down 2024-11-25T12:45:04,179 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:04,182 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T12:45:04,213 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T12:45:04,215 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:04,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741841_1017 (size=469) 2024-11-25T12:45:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741841_1017 (size=469) 2024-11-25T12:45:04,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741841_1017 (size=469) 2024-11-25T12:45:04,234 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:04,240 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002 2024-11-25T12:45:04,244 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-25T12:45:04,244 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1, journal: Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1, size=469 (469bytes) at 1732538704127Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1 so closing down at 1732538704179 (+52 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp at 1732538704213 (+34 ms)3 split writer threads finished at 1732538704215 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538704234 (+19 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002 at 1732538704240 (+6 ms)Processed 2 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732538704244 (+4 ms) 2024-11-25T12:45:04,262 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2, size=322 (322bytes) 2024-11-25T12:45:04,262 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2 2024-11-25T12:45:04,263 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2 after 1ms 2024-11-25T12:45:04,307 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:04,307 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2 took 46ms 2024-11-25T12:45:04,318 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2 so closing down 2024-11-25T12:45:04,319 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:04,322 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T12:45:04,362 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T12:45:04,362 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:04,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741842_1018 (size=322) 2024-11-25T12:45:04,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741842_1018 (size=322) 2024-11-25T12:45:04,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741842_1018 (size=322) 2024-11-25T12:45:04,377 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:04,411 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:04,414 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-25T12:45:04,417 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-25T12:45:04,417 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2, journal: Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2, size=322 (322bytes) at 1732538704262Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2 so closing down at 1732538704318 (+56 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002-wal-2.temp at 1732538704362 (+44 ms)3 split writer threads finished at 1732538704362Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538704377 (+15 ms)Processed 1 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732538704417 (+40 ms) 2024-11-25T12:45:04,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:04,421 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:04,440 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal.1732538704422, exclude list is [], retry=0 2024-11-25T12:45:04,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:04,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:04,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:04,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal.1732538704422 2024-11-25T12:45:04,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:04,484 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e4249dccf225607b1521778c17a5c90, NAME => 'testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:04,485 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:04,485 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,485 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,488 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,489 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e4249dccf225607b1521778c17a5c90 columnFamilyName a 2024-11-25T12:45:04,489 DEBUG [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:04,490 INFO [StoreOpener-3e4249dccf225607b1521778c17a5c90-1 {}] regionserver.HStore(327): Store=3e4249dccf225607b1521778c17a5c90/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:04,491 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,498 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002 2024-11-25T12:45:04,531 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:04,542 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002 2024-11-25T12:45:04,547 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3e4249dccf225607b1521778c17a5c90 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T12:45:04,630 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/.tmp/a/51fb58db20e543c5941c9889a1c43a77 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732538703901/Put/seqid=0 2024-11-25T12:45:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:45:04,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:45:04,684 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/.tmp/a/51fb58db20e543c5941c9889a1c43a77 2024-11-25T12:45:04,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741844_1020 (size=5170) 2024-11-25T12:45:04,747 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/.tmp/a/51fb58db20e543c5941c9889a1c43a77 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/a/51fb58db20e543c5941c9889a1c43a77 2024-11-25T12:45:04,782 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/a/51fb58db20e543c5941c9889a1c43a77, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T12:45:04,794 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3e4249dccf225607b1521778c17a5c90 in 242ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T12:45:04,796 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/0000000000000000002 2024-11-25T12:45:04,797 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,797 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,801 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3e4249dccf225607b1521778c17a5c90 2024-11-25T12:45:04,805 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/3e4249dccf225607b1521778c17a5c90/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T12:45:04,807 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3e4249dccf225607b1521778c17a5c90; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59588405, jitterRate=-0.11206357181072235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:04,808 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3e4249dccf225607b1521778c17a5c90: Writing region info on filesystem at 1732538704485Initializing all the Stores at 1732538704487 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538704487Obtaining lock to block concurrent updates at 1732538704547 (+60 ms)Preparing flush snapshotting stores in 3e4249dccf225607b1521778c17a5c90 at 1732538704547Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732538704551 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732538703677.3e4249dccf225607b1521778c17a5c90. at 1732538704551Flushing 3e4249dccf225607b1521778c17a5c90/a: creating writer at 1732538704553 (+2 ms)Flushing 3e4249dccf225607b1521778c17a5c90/a: appending metadata at 1732538704610 (+57 ms)Flushing 3e4249dccf225607b1521778c17a5c90/a: closing flushed file at 1732538704614 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19646e72: reopening flushed file at 1732538704745 (+131 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3e4249dccf225607b1521778c17a5c90 in 242ms, sequenceid=2, compaction requested=false; wal=null at 1732538704794 (+49 ms)Cleaning up temporary data from old regions at 1732538704797 (+3 ms)Region opened successfully at 1732538704808 (+11 ms) 2024-11-25T12:45:04,841 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 367) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:50406 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53800 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:57022 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41185/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:54500 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53834 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=2612 (was 2641) 2024-11-25T12:45:04,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=2608 2024-11-25T12:45:04,879 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:04,881 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:04,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:04,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12691879, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-12691879, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:04,901 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12691879/hregion-12691879.1732538704887, exclude list is [], retry=0 2024-11-25T12:45:04,909 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:04,910 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:04,916 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:04,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12691879/hregion-12691879.1732538704887 2024-11-25T12:45:04,955 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:04,956 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d1aea379fc2b640a5de3f16fbce01d3, NAME => 'testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:45:05,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:45:05,018 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:05,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741846_1022 (size=64) 2024-11-25T12:45:05,025 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,036 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d1aea379fc2b640a5de3f16fbce01d3 columnFamilyName a 2024-11-25T12:45:05,036 DEBUG [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:05,037 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] regionserver.HStore(327): Store=2d1aea379fc2b640a5de3f16fbce01d3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:05,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,040 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,041 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,041 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,046 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,054 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:05,056 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d1aea379fc2b640a5de3f16fbce01d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67705823, jitterRate=0.008895382285118103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:05,056 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d1aea379fc2b640a5de3f16fbce01d3: Writing region info on filesystem at 1732538705019Initializing all the Stores at 1732538705021 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538705022 (+1 ms)Cleaning up temporary data from old regions at 1732538705042 (+20 ms)Region opened successfully at 1732538705056 (+14 ms) 2024-11-25T12:45:05,056 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d1aea379fc2b640a5de3f16fbce01d3, disabling compactions & flushes 2024-11-25T12:45:05,056 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. 2024-11-25T12:45:05,056 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. 2024-11-25T12:45:05,056 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. after waiting 0 ms 2024-11-25T12:45:05,057 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. 2024-11-25T12:45:05,059 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. 2024-11-25T12:45:05,060 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d1aea379fc2b640a5de3f16fbce01d3: Waiting for close lock at 1732538705056Disabling compacts and flushes for region at 1732538705056Disabling writes for close at 1732538705057 (+1 ms)Writing region close event to WAL at 1732538705059 (+2 ms)Closed at 1732538705059 2024-11-25T12:45:05,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741845_1021 (size=175) 2024-11-25T12:45:05,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741845_1021 (size=175) 2024-11-25T12:45:05,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741845_1021 (size=175) 2024-11-25T12:45:05,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:05,077 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12691879:(num 1732538704887) 2024-11-25T12:45:05,079 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:45:05,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741847_1023 (size=469) 2024-11-25T12:45:05,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741847_1023 (size=469) 2024-11-25T12:45:05,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741847_1023 (size=469) 2024-11-25T12:45:05,186 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T12:45:05,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741848_1024 (size=322) 2024-11-25T12:45:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741848_1024 (size=322) 2024-11-25T12:45:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741848_1024 (size=322) 2024-11-25T12:45:05,262 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2, size=322 (322bytes) 2024-11-25T12:45:05,262 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2 2024-11-25T12:45:05,263 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2 after 1ms 2024-11-25T12:45:05,291 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:05,292 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2 took 31ms 2024-11-25T12:45:05,295 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2 so closing down 2024-11-25T12:45:05,295 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:05,299 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T12:45:05,326 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T12:45:05,327 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741849_1025 (size=322) 2024-11-25T12:45:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741849_1025 (size=322) 2024-11-25T12:45:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741849_1025 (size=322) 2024-11-25T12:45:05,348 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:05,351 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 2024-11-25T12:45:05,352 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-25T12:45:05,352 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2, journal: Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2, size=322 (322bytes) at 1732538705262Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2 so closing down at 1732538705295 (+33 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp at 1732538705326 (+31 ms)3 split writer threads finished at 1732538705327 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538705349 (+22 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 at 1732538705351 (+2 ms)Processed 1 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732538705352 (+1 ms) 2024-11-25T12:45:05,371 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1, size=469 (469bytes) 2024-11-25T12:45:05,371 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1 2024-11-25T12:45:05,372 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1 after 1ms 2024-11-25T12:45:05,403 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:05,403 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1 took 33ms 2024-11-25T12:45:05,407 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1 so closing down 2024-11-25T12:45:05,407 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:05,410 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T12:45:05,442 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T12:45:05,442 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:05,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741850_1026 (size=469) 2024-11-25T12:45:05,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741850_1026 (size=469) 2024-11-25T12:45:05,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741850_1026 (size=469) 2024-11-25T12:45:05,453 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:05,486 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:05,489 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002, length=322 2024-11-25T12:45:05,491 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 2024-11-25T12:45:05,492 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-25T12:45:05,492 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1, journal: Splitting hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1, size=469 (469bytes) at 1732538705371Finishing writing output for hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1 so closing down at 1732538705407 (+36 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp at 1732538705442 (+35 ms)3 split writer threads finished at 1732538705442Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538705453 (+11 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 at 1732538705492 (+39 ms)Processed 2 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732538705492 2024-11-25T12:45:05,492 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:05,494 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:05,511 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal.1732538705495, exclude list is [], retry=0 2024-11-25T12:45:05,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:05,516 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:05,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:05,546 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal.1732538705495 2024-11-25T12:45:05,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:05,548 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d1aea379fc2b640a5de3f16fbce01d3, NAME => 'testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,556 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,558 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d1aea379fc2b640a5de3f16fbce01d3 columnFamilyName a 2024-11-25T12:45:05,558 DEBUG [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:05,560 INFO [StoreOpener-2d1aea379fc2b640a5de3f16fbce01d3-1 {}] regionserver.HStore(327): Store=2d1aea379fc2b640a5de3f16fbce01d3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:05,560 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,565 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,566 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 2024-11-25T12:45:05,600 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:05,602 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 2024-11-25T12:45:05,603 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d1aea379fc2b640a5de3f16fbce01d3 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T12:45:05,621 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/.tmp/a/8861b20712b049f88b3b91d8966b6412 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732538705077/Put/seqid=0 2024-11-25T12:45:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:45:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:45:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741852_1028 (size=5170) 2024-11-25T12:45:05,635 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/.tmp/a/8861b20712b049f88b3b91d8966b6412 2024-11-25T12:45:05,649 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/.tmp/a/8861b20712b049f88b3b91d8966b6412 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/a/8861b20712b049f88b3b91d8966b6412 2024-11-25T12:45:05,666 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/a/8861b20712b049f88b3b91d8966b6412, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T12:45:05,667 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2d1aea379fc2b640a5de3f16fbce01d3 in 64ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T12:45:05,668 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/0000000000000000002 2024-11-25T12:45:05,668 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,669 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,671 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d1aea379fc2b640a5de3f16fbce01d3 2024-11-25T12:45:05,675 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/2d1aea379fc2b640a5de3f16fbce01d3/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T12:45:05,676 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d1aea379fc2b640a5de3f16fbce01d3; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71873368, jitterRate=0.07099664211273193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:05,677 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d1aea379fc2b640a5de3f16fbce01d3: Writing region info on filesystem at 1732538705549Initializing all the Stores at 1732538705551 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538705551Obtaining lock to block concurrent updates at 1732538705603 (+52 ms)Preparing flush snapshotting stores in 2d1aea379fc2b640a5de3f16fbce01d3 at 1732538705603Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732538705603Flushing stores of testReplayEditsWrittenIntoWAL,,1732538704880.2d1aea379fc2b640a5de3f16fbce01d3. at 1732538705603Flushing 2d1aea379fc2b640a5de3f16fbce01d3/a: creating writer at 1732538705603Flushing 2d1aea379fc2b640a5de3f16fbce01d3/a: appending metadata at 1732538705620 (+17 ms)Flushing 2d1aea379fc2b640a5de3f16fbce01d3/a: closing flushed file at 1732538705620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249a9509: reopening flushed file at 1732538705647 (+27 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2d1aea379fc2b640a5de3f16fbce01d3 in 64ms, sequenceid=2, compaction requested=false; wal=null at 1732538705667 (+20 ms)Cleaning up temporary data from old regions at 1732538705669 (+2 ms)Region opened successfully at 1732538705677 (+8 ms) 2024-11-25T12:45:05,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:50500 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:54562 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53908 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:42778 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53800 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=2611 (was 2608) - AvailableMemoryMB LEAK? - 2024-11-25T12:45:05,719 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=2609 2024-11-25T12:45:05,744 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:05,748 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:05,750 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:05,759 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-03398075, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-03398075, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:05,778 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-03398075/hregion-03398075.1732538705759, exclude list is [], retry=0 2024-11-25T12:45:05,783 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:05,787 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:05,789 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:05,820 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-03398075/hregion-03398075.1732538705759 2024-11-25T12:45:05,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:05,820 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4f9fdef32cab05390533bf5d414616de, NAME => 'testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:45:05,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:45:05,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741854_1030 (size=64) 2024-11-25T12:45:05,848 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:05,854 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,860 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName a 2024-11-25T12:45:05,860 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:05,861 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:05,861 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,866 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName b 2024-11-25T12:45:05,866 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:05,867 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:05,867 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,869 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName c 2024-11-25T12:45:05,869 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:05,870 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:05,870 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,871 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,873 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,874 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:05,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:05,880 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:05,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4f9fdef32cab05390533bf5d414616de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59881988, jitterRate=-0.10768884420394897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:05,881 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4f9fdef32cab05390533bf5d414616de: Writing region info on filesystem at 1732538705851Initializing all the Stores at 1732538705853 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538705853Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538705854 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538705854Cleaning up temporary data from old regions at 1732538705874 (+20 ms)Region opened successfully at 1732538705881 (+7 ms) 2024-11-25T12:45:05,881 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4f9fdef32cab05390533bf5d414616de, disabling compactions & flushes 2024-11-25T12:45:05,881 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:05,881 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:05,882 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. after waiting 0 ms 2024-11-25T12:45:05,882 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:05,883 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:05,884 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4f9fdef32cab05390533bf5d414616de: Waiting for close lock at 1732538705881Disabling compacts and flushes for region at 1732538705881Disabling writes for close at 1732538705882 (+1 ms)Writing region close event to WAL at 1732538705883 (+1 ms)Closed at 1732538705883 2024-11-25T12:45:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741853_1029 (size=175) 2024-11-25T12:45:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741853_1029 (size=175) 2024-11-25T12:45:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741853_1029 (size=175) 2024-11-25T12:45:05,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:05,911 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-03398075:(num 1732538705759) 2024-11-25T12:45:05,911 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:05,920 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:05,941 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, exclude list is [], retry=0 2024-11-25T12:45:05,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:05,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:05,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:05,986 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 2024-11-25T12:45:05,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:06,283 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, size=0 (0bytes) 2024-11-25T12:45:06,283 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 might be still open, length is 0 2024-11-25T12:45:06,283 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 2024-11-25T12:45:06,285 WARN [IPC Server handler 4 on default port 41185 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-25T12:45:06,286 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 after 3ms 2024-11-25T12:45:07,015 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:54610 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35695:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54610 dst: /127.0.0.1:35695 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35695 remote=/127.0.0.1:54610]. Total timeout mills is 60000, 59215 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:07,017 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:50532 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45631:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50532 dst: /127.0.0.1:45631 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:07,017 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53956 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53956 dst: /127.0.0.1:38775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:07,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741855_1032 (size=497775) 2024-11-25T12:45:07,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741855_1032 (size=497775) 2024-11-25T12:45:07,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741855_1032 (size=497775) 2024-11-25T12:45:08,625 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:45:08,694 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T12:45:09,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T12:45:09,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:09,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:45:09,590 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T12:45:09,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T12:45:09,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:09,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-25T12:45:09,592 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:10,287 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 after 4004ms 2024-11-25T12:45:10,334 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:10,340 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 took 4057ms 2024-11-25T12:45:10,358 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732538705921.temp 2024-11-25T12:45:10,403 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp 2024-11-25T12:45:10,494 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921; continuing. 2024-11-25T12:45:10,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 so closing down 2024-11-25T12:45:10,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:10,496 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:10,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741856_1033 (size=497783) 2024-11-25T12:45:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741856_1033 (size=497783) 2024-11-25T12:45:10,518 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp (wrote 3002 edits, skipped 0 edits in 48 ms) 2024-11-25T12:45:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741856_1033 (size=497783) 2024-11-25T12:45:10,522 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 2024-11-25T12:45:10,523 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 182 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:45:10,523 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, journal: Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, size=0 (0bytes) at 1732538706283Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp at 1732538710403 (+4120 ms)Split 1024 edits, skipped 0 edits. at 1732538710443 (+40 ms)Split 2048 edits, skipped 0 edits. at 1732538710471 (+28 ms)Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 so closing down at 1732538710494 (+23 ms)3 split writer threads finished at 1732538710496 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp (wrote 3002 edits, skipped 0 edits in 48 ms) at 1732538710518 (+22 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000000001-wal.1732538705921.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 at 1732538710522 (+4 ms)Processed 3002 edits across 1 Regions in 182 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921, size=0, length=0, corrupted=false, cancelled=false at 1732538710523 (+1 ms) 2024-11-25T12:45:10,526 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538705921 2024-11-25T12:45:10,531 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 2024-11-25T12:45:10,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:10,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:10,555 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538710536, exclude list is [], retry=0 2024-11-25T12:45:10,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:10,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:10,561 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:10,628 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538710536 2024-11-25T12:45:10,631 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:10,632 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:10,648 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,652 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName a 2024-11-25T12:45:10,652 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:10,654 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:10,654 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,657 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName b 2024-11-25T12:45:10,657 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:10,658 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:10,659 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,667 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f9fdef32cab05390533bf5d414616de columnFamilyName c 2024-11-25T12:45:10,668 DEBUG [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:10,676 INFO [StoreOpener-4f9fdef32cab05390533bf5d414616de-1 {}] regionserver.HStore(327): Store=4f9fdef32cab05390533bf5d414616de/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:10,676 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:10,686 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 2024-11-25T12:45:10,737 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:10,811 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:45:11,438 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f9fdef32cab05390533bf5d414616de 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:45:11,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/282639e7a4ac448c8ec9fc87fcdc199f is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732538705997/Put/seqid=0 2024-11-25T12:45:11,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:45:11,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:45:11,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741858_1035 (size=50463) 2024-11-25T12:45:11,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/282639e7a4ac448c8ec9fc87fcdc199f 2024-11-25T12:45:11,563 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/282639e7a4ac448c8ec9fc87fcdc199f as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/a/282639e7a4ac448c8ec9fc87fcdc199f 2024-11-25T12:45:11,575 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/a/282639e7a4ac448c8ec9fc87fcdc199f, entries=754, sequenceid=754, filesize=49.3 K 2024-11-25T12:45:11,575 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 4f9fdef32cab05390533bf5d414616de in 137ms, sequenceid=754, compaction requested=false; wal=null 2024-11-25T12:45:11,602 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:45:11,603 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f9fdef32cab05390533bf5d414616de 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:45:11,618 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/0ad9547a45f34cbfb2ddd8a55b6cb26e is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732538706057/Put/seqid=0 2024-11-25T12:45:11,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:45:11,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:45:11,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741859_1036 (size=20072) 2024-11-25T12:45:11,645 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/0ad9547a45f34cbfb2ddd8a55b6cb26e 2024-11-25T12:45:11,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/d2a8bc5cd9d7478a8459488807ecbf78 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732538706086/Put/seqid=0 2024-11-25T12:45:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:45:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:45:11,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741860_1037 (size=35835) 2024-11-25T12:45:11,686 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/d2a8bc5cd9d7478a8459488807ecbf78 2024-11-25T12:45:11,694 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/a/0ad9547a45f34cbfb2ddd8a55b6cb26e as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/a/0ad9547a45f34cbfb2ddd8a55b6cb26e 2024-11-25T12:45:11,703 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/a/0ad9547a45f34cbfb2ddd8a55b6cb26e, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-25T12:45:11,705 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/d2a8bc5cd9d7478a8459488807ecbf78 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/b/d2a8bc5cd9d7478a8459488807ecbf78 2024-11-25T12:45:11,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/b/d2a8bc5cd9d7478a8459488807ecbf78, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-25T12:45:11,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4f9fdef32cab05390533bf5d414616de in 110ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-25T12:45:11,731 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:45:11,732 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f9fdef32cab05390533bf5d414616de 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T12:45:11,744 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/c9792081da89434aafea761b2366d68e is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732538706113/Put/seqid=0 2024-11-25T12:45:11,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:45:11,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:45:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741861_1038 (size=35082) 2024-11-25T12:45:11,772 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/c9792081da89434aafea761b2366d68e 2024-11-25T12:45:11,797 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/9d49d1814702415f80c1d64090e6030c is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732538706159/Put/seqid=0 2024-11-25T12:45:11,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:45:11,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:45:11,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741862_1039 (size=20825) 2024-11-25T12:45:11,809 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/9d49d1814702415f80c1d64090e6030c 2024-11-25T12:45:11,822 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/b/c9792081da89434aafea761b2366d68e as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/b/c9792081da89434aafea761b2366d68e 2024-11-25T12:45:11,830 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/b/c9792081da89434aafea761b2366d68e, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-25T12:45:11,832 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/9d49d1814702415f80c1d64090e6030c as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/c/9d49d1814702415f80c1d64090e6030c 2024-11-25T12:45:11,840 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/c/9d49d1814702415f80c1d64090e6030c, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-25T12:45:11,840 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4f9fdef32cab05390533bf5d414616de in 108ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-25T12:45:11,853 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732538706228/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:11,858 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 2024-11-25T12:45:11,858 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T12:45:11,858 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4f9fdef32cab05390533bf5d414616de 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-25T12:45:11,873 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/0d45cf0975484c0fa6b0290f33111707 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732538706171/Put/seqid=0 2024-11-25T12:45:11,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:45:11,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:45:11,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741863_1040 (size=50301) 2024-11-25T12:45:11,896 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/0d45cf0975484c0fa6b0290f33111707 2024-11-25T12:45:11,906 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d45cf0975484c0fa6b0290f33111707 2024-11-25T12:45:11,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/.tmp/c/0d45cf0975484c0fa6b0290f33111707 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/c/0d45cf0975484c0fa6b0290f33111707 2024-11-25T12:45:11,917 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d45cf0975484c0fa6b0290f33111707 2024-11-25T12:45:11,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/c/0d45cf0975484c0fa6b0290f33111707, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-25T12:45:11,917 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 4f9fdef32cab05390533bf5d414616de in 59ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-25T12:45:11,919 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/0000000000000003002 2024-11-25T12:45:11,921 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:11,921 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:11,923 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:45:11,926 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4f9fdef32cab05390533bf5d414616de 2024-11-25T12:45:11,930 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenIntoWAL/4f9fdef32cab05390533bf5d414616de/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-25T12:45:11,931 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4f9fdef32cab05390533bf5d414616de; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67402197, jitterRate=0.0043710023164749146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:45:11,932 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4f9fdef32cab05390533bf5d414616de: Writing region info on filesystem at 1732538710632Initializing all the Stores at 1732538710645 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538710645Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538710647 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538710647Cleaning up temporary data from old regions at 1732538711921 (+1274 ms)Region opened successfully at 1732538711932 (+11 ms) 2024-11-25T12:45:12,009 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4f9fdef32cab05390533bf5d414616de, disabling compactions & flushes 2024-11-25T12:45:12,009 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:12,009 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:12,010 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. after waiting 0 ms 2024-11-25T12:45:12,010 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:12,017 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732538705745.4f9fdef32cab05390533bf5d414616de. 2024-11-25T12:45:12,017 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4f9fdef32cab05390533bf5d414616de: Waiting for close lock at 1732538712009Disabling compacts and flushes for region at 1732538712009Disabling writes for close at 1732538712010 (+1 ms)Writing region close event to WAL at 1732538712017 (+7 ms)Closed at 1732538712017 2024-11-25T12:45:12,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741857_1034 (size=175) 2024-11-25T12:45:12,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741857_1034 (size=175) 2024-11-25T12:45:12,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741857_1034 (size=175) 2024-11-25T12:45:12,024 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538710536 not finished, retry = 0 2024-11-25T12:45:12,128 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:12,128 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538710536) 2024-11-25T12:45:12,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 388) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36989 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41185 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:34915 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1447706172_22 at /127.0.0.1:54658 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1447706172_22 at /127.0.0.1:50594 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34915 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1447706172_22 at /127.0.0.1:50570 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36989 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1447706172_22 at /127.0.0.1:53994 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 517) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2442 (was 2609) 2024-11-25T12:45:12,167 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=2439 2024-11-25T12:45:12,190 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:12,195 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:12,197 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:12,204 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72405777, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-72405777, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:12,223 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72405777/hregion-72405777.1732538712205, exclude list is [], retry=0 2024-11-25T12:45:12,227 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:12,229 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:12,231 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:12,275 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72405777/hregion-72405777.1732538712205 2024-11-25T12:45:12,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:12,276 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 586891001dcd287b83e66c6b44203a50, NAME => 'test2727,,1732538712191.586891001dcd287b83e66c6b44203a50.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:45:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:45:12,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741865_1042 (size=43) 2024-11-25T12:45:12,289 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732538712191.586891001dcd287b83e66c6b44203a50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:12,291 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,293 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName a 2024-11-25T12:45:12,293 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:12,294 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:12,294 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,296 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName b 2024-11-25T12:45:12,296 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:12,297 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:12,297 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,300 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName c 2024-11-25T12:45:12,300 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:12,303 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:12,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,307 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,307 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,308 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:12,315 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:12,324 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:12,325 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 586891001dcd287b83e66c6b44203a50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74550536, jitterRate=0.11088955402374268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:12,326 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 586891001dcd287b83e66c6b44203a50: Writing region info on filesystem at 1732538712290Initializing all the Stores at 1732538712290Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538712291 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538712291Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538712291Cleaning up temporary data from old regions at 1732538712307 (+16 ms)Region opened successfully at 1732538712326 (+19 ms) 2024-11-25T12:45:12,326 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 586891001dcd287b83e66c6b44203a50, disabling compactions & flushes 2024-11-25T12:45:12,326 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:12,327 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:12,327 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. after waiting 0 ms 2024-11-25T12:45:12,327 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:12,332 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:12,332 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 586891001dcd287b83e66c6b44203a50: Waiting for close lock at 1732538712326Disabling compacts and flushes for region at 1732538712326Disabling writes for close at 1732538712327 (+1 ms)Writing region close event to WAL at 1732538712331 (+4 ms)Closed at 1732538712331 2024-11-25T12:45:12,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741864_1041 (size=175) 2024-11-25T12:45:12,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741864_1041 (size=175) 2024-11-25T12:45:12,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741864_1041 (size=175) 2024-11-25T12:45:12,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:12,345 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72405777:(num 1732538712205) 2024-11-25T12:45:12,345 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:12,347 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:12,364 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, exclude list is [], retry=0 2024-11-25T12:45:12,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:12,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:12,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:12,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 2024-11-25T12:45:12,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:12,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741866_1043 (size=371388) 2024-11-25T12:45:12,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741866_1043 (size=371388) 2024-11-25T12:45:12,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741866_1043 (size=371388) 2024-11-25T12:45:12,715 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, size=362.7 K (371388bytes) 2024-11-25T12:45:12,715 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 2024-11-25T12:45:12,716 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 after 1ms 2024-11-25T12:45:12,769 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:12,770 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 took 56ms 2024-11-25T12:45:12,776 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732538712348.temp 2024-11-25T12:45:12,816 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 so closing down 2024-11-25T12:45:12,816 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:12,819 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp 2024-11-25T12:45:12,836 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:12,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741867_1044 (size=371388) 2024-11-25T12:45:12,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741867_1044 (size=371388) 2024-11-25T12:45:12,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741867_1044 (size=371388) 2024-11-25T12:45:12,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-25T12:45:12,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp to hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 2024-11-25T12:45:12,849 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-25T12:45:12,849 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, journal: Splitting hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, size=362.7 K (371388bytes) at 1732538712715Split 1024 edits, skipped 0 edits. at 1732538712788 (+73 ms)Split 2048 edits, skipped 0 edits. at 1732538712803 (+15 ms)Finishing writing output for hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 so closing down at 1732538712816 (+13 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp at 1732538712819 (+3 ms)3 split writer threads finished at 1732538712836 (+17 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732538712846 (+10 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000000001-wal.1732538712348.temp to hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 at 1732538712849 (+3 ms)Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732538712849 2024-11-25T12:45:12,851 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712348 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538712348 2024-11-25T12:45:12,853 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 2024-11-25T12:45:12,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:12,857 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:12,870 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, exclude list is [], retry=0 2024-11-25T12:45:12,874 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:12,874 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:12,875 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:12,910 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 2024-11-25T12:45:12,910 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741868_1045 (size=371515) 2024-11-25T12:45:13,069 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 not finished, retry = 0 2024-11-25T12:45:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741868_1045 (size=371515) 2024-11-25T12:45:13,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741868_1045 (size=371515) 2024-11-25T12:45:13,191 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, size=362.8 K (371515bytes) 2024-11-25T12:45:13,191 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 2024-11-25T12:45:13,192 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 after 1ms 2024-11-25T12:45:13,237 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:13,239 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 took 48ms 2024-11-25T12:45:13,284 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732538712858.temp 2024-11-25T12:45:13,363 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 so closing down 2024-11-25T12:45:13,363 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:13,363 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp 2024-11-25T12:45:13,380 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:13,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741869_1046 (size=371515) 2024-11-25T12:45:13,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741869_1046 (size=371515) 2024-11-25T12:45:13,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741869_1046 (size=371515) 2024-11-25T12:45:13,403 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-25T12:45:13,406 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp to hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 2024-11-25T12:45:13,406 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 164 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-25T12:45:13,406 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, journal: Splitting hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, size=362.8 K (371515bytes) at 1732538713191Split 1024 edits, skipped 0 edits. at 1732538713301 (+110 ms)Split 2048 edits, skipped 0 edits. at 1732538713351 (+50 ms)Finishing writing output for hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 so closing down at 1732538713363 (+12 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp at 17325387133633 split writer threads finished at 1732538713380 (+17 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732538713403 (+23 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003001-wal.1732538712858.temp to hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 at 1732538713406 (+3 ms)Processed 3000 edits across 1 Regions in 164 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732538713406 2024-11-25T12:45:13,412 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538712858 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538712858 2024-11-25T12:45:13,413 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 2024-11-25T12:45:13,414 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:13,416 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/test2727-manual,16010,1732538712189, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:13,435 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538713417, exclude list is [], retry=0 2024-11-25T12:45:13,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:13,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:13,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:13,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732538712189/wal.1732538713417 2024-11-25T12:45:13,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:13,496 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 586891001dcd287b83e66c6b44203a50, NAME => 'test2727,,1732538712191.586891001dcd287b83e66c6b44203a50.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:13,496 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732538712191.586891001dcd287b83e66c6b44203a50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:13,496 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,496 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,499 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,501 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName a 2024-11-25T12:45:13,501 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:13,502 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:13,502 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,503 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName b 2024-11-25T12:45:13,503 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:13,504 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:13,504 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,505 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 586891001dcd287b83e66c6b44203a50 columnFamilyName c 2024-11-25T12:45:13,505 DEBUG [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:13,506 INFO [StoreOpener-586891001dcd287b83e66c6b44203a50-1 {}] regionserver.HStore(327): Store=586891001dcd287b83e66c6b44203a50/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:13,506 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,507 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:13,511 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 2024-11-25T12:45:13,550 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:13,595 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 2024-11-25T12:45:13,598 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 2024-11-25T12:45:13,641 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:13,677 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 2024-11-25T12:45:13,678 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 586891001dcd287b83e66c6b44203a50 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-25T12:45:13,717 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/a/42b5c8d115db43daba2dd03f6c2e1d25 is 41, key is test2727/a:100/1732538712917/Put/seqid=0 2024-11-25T12:45:13,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:45:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:45:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741871_1048 (size=84227) 2024-11-25T12:45:13,744 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/a/42b5c8d115db43daba2dd03f6c2e1d25 2024-11-25T12:45:13,817 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/b/1ead29bdfc0545b4b0933fb77b4af285 is 41, key is test2727/b:100/1732538712967/Put/seqid=0 2024-11-25T12:45:13,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:45:13,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:45:13,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741872_1049 (size=84609) 2024-11-25T12:45:13,846 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/b/1ead29bdfc0545b4b0933fb77b4af285 2024-11-25T12:45:13,904 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/c/2e075a84d2e74625aa394b00b07d63a4 is 41, key is test2727/c:100/1732538713023/Put/seqid=0 2024-11-25T12:45:13,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:45:13,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:45:13,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741873_1050 (size=84609) 2024-11-25T12:45:13,959 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/c/2e075a84d2e74625aa394b00b07d63a4 2024-11-25T12:45:13,968 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/a/42b5c8d115db43daba2dd03f6c2e1d25 as hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/a/42b5c8d115db43daba2dd03f6c2e1d25 2024-11-25T12:45:13,979 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/a/42b5c8d115db43daba2dd03f6c2e1d25, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-25T12:45:13,981 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/b/1ead29bdfc0545b4b0933fb77b4af285 as hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/b/1ead29bdfc0545b4b0933fb77b4af285 2024-11-25T12:45:13,993 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/b/1ead29bdfc0545b4b0933fb77b4af285, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T12:45:14,007 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/.tmp/c/2e075a84d2e74625aa394b00b07d63a4 as hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/c/2e075a84d2e74625aa394b00b07d63a4 2024-11-25T12:45:14,028 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/c/2e075a84d2e74625aa394b00b07d63a4, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T12:45:14,028 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 586891001dcd287b83e66c6b44203a50 in 351ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-25T12:45:14,029 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000003000 2024-11-25T12:45:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/0000000000000006000 2024-11-25T12:45:14,033 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:14,033 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:14,035 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:14,043 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 586891001dcd287b83e66c6b44203a50 2024-11-25T12:45:14,048 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/test2727/586891001dcd287b83e66c6b44203a50/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-25T12:45:14,049 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 586891001dcd287b83e66c6b44203a50; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66931065, jitterRate=-0.0026494115591049194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:14,050 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 586891001dcd287b83e66c6b44203a50: Writing region info on filesystem at 1732538713496Initializing all the Stores at 1732538713498 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538713498Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538713499 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538713499Obtaining lock to block concurrent updates at 1732538713678 (+179 ms)Preparing flush snapshotting stores in 586891001dcd287b83e66c6b44203a50 at 1732538713678Finished memstore snapshotting test2727,,1732538712191.586891001dcd287b83e66c6b44203a50., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732538713678Flushing stores of test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. at 1732538713678Flushing 586891001dcd287b83e66c6b44203a50/a: creating writer at 1732538713678Flushing 586891001dcd287b83e66c6b44203a50/a: appending metadata at 1732538713716 (+38 ms)Flushing 586891001dcd287b83e66c6b44203a50/a: closing flushed file at 1732538713716Flushing 586891001dcd287b83e66c6b44203a50/b: creating writer at 1732538713765 (+49 ms)Flushing 586891001dcd287b83e66c6b44203a50/b: appending metadata at 1732538713807 (+42 ms)Flushing 586891001dcd287b83e66c6b44203a50/b: closing flushed file at 1732538713815 (+8 ms)Flushing 586891001dcd287b83e66c6b44203a50/c: creating writer at 1732538713869 (+54 ms)Flushing 586891001dcd287b83e66c6b44203a50/c: appending metadata at 1732538713903 (+34 ms)Flushing 586891001dcd287b83e66c6b44203a50/c: closing flushed file at 1732538713904 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@164cedd8: reopening flushed file at 1732538713967 (+63 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62f65dcf: reopening flushed file at 1732538713979 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ab71a0: reopening flushed file at 1732538713994 (+15 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 586891001dcd287b83e66c6b44203a50 in 351ms, sequenceid=6000, compaction requested=false; wal=null at 1732538714028 (+34 ms)Cleaning up temporary data from old regions at 1732538714033 (+5 ms)Region opened successfully at 1732538714050 (+17 ms) 2024-11-25T12:45:14,052 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-25T12:45:14,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 586891001dcd287b83e66c6b44203a50, disabling compactions & flushes 2024-11-25T12:45:14,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:14,052 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:14,052 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. after waiting 0 ms 2024-11-25T12:45:14,052 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:14,064 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732538712191.586891001dcd287b83e66c6b44203a50. 2024-11-25T12:45:14,065 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 586891001dcd287b83e66c6b44203a50: Waiting for close lock at 1732538714052Disabling compacts and flushes for region at 1732538714052Disabling writes for close at 1732538714052Writing region close event to WAL at 1732538714064 (+12 ms)Closed at 1732538714064 2024-11-25T12:45:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741870_1047 (size=175) 2024-11-25T12:45:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741870_1047 (size=175) 2024-11-25T12:45:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741870_1047 (size=175) 2024-11-25T12:45:14,082 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:14,082 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538713417) 2024-11-25T12:45:14,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404 (was 404), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 524) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2139 (was 2439) 2024-11-25T12:45:14,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=546, ProcessCount=11, AvailableMemoryMB=2138 2024-11-25T12:45:14,132 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:14,138 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:14,139 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732538714139 2024-11-25T12:45:14,183 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 2024-11-25T12:45:14,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:14,193 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a1a6b17902032562890f818c81af74ba, NAME => 'testSequentialEditLogSeqNum,,1732538714133.a1a6b17902032562890f818c81af74ba.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:14,193 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732538714133.a1a6b17902032562890f818c81af74ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:14,193 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,193 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,195 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba doesn't exist for region: a1a6b17902032562890f818c81af74ba on table testSequentialEditLogSeqNum 2024-11-25T12:45:14,195 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: a1a6b17902032562890f818c81af74ba on table testSequentialEditLogSeqNum 2024-11-25T12:45:14,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:45:14,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:45:14,216 INFO [StoreOpener-a1a6b17902032562890f818c81af74ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741875_1052 (size=62) 2024-11-25T12:45:14,223 INFO [StoreOpener-a1a6b17902032562890f818c81af74ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1a6b17902032562890f818c81af74ba columnFamilyName a 2024-11-25T12:45:14,223 DEBUG [StoreOpener-a1a6b17902032562890f818c81af74ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:14,224 INFO [StoreOpener-a1a6b17902032562890f818c81af74ba-1 {}] regionserver.HStore(327): Store=a1a6b17902032562890f818c81af74ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:14,224 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,227 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,227 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,233 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1a6b17902032562890f818c81af74ba 2024-11-25T12:45:14,244 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:14,245 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1a6b17902032562890f818c81af74ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66900481, jitterRate=-0.003105148673057556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:14,246 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1a6b17902032562890f818c81af74ba: Writing region info on filesystem at 1732538714194Initializing all the Stores at 1732538714215 (+21 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538714215Cleaning up temporary data from old regions at 1732538714228 (+13 ms)Region opened successfully at 1732538714246 (+18 ms) 2024-11-25T12:45:14,274 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a1a6b17902032562890f818c81af74ba 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-25T12:45:14,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/.tmp/a/8af93a46dd2e4913b3a887cba627e70d is 81, key is testSequentialEditLogSeqNum/a:x0/1732538714246/Put/seqid=0 2024-11-25T12:45:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:45:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:45:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741876_1053 (size=5833) 2024-11-25T12:45:14,756 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/.tmp/a/8af93a46dd2e4913b3a887cba627e70d 2024-11-25T12:45:14,768 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/.tmp/a/8af93a46dd2e4913b3a887cba627e70d as hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/a/8af93a46dd2e4913b3a887cba627e70d 2024-11-25T12:45:14,778 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/a/8af93a46dd2e4913b3a887cba627e70d, entries=10, sequenceid=13, filesize=5.7 K 2024-11-25T12:45:14,780 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a1a6b17902032562890f818c81af74ba in 506ms, sequenceid=13, compaction requested=false 2024-11-25T12:45:14,781 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a1a6b17902032562890f818c81af74ba: 2024-11-25T12:45:14,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:45:14,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:45:14,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:45:14,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:45:14,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T12:45:14,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741874_1051 (size=3256) 2024-11-25T12:45:14,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741874_1051 (size=3256) 2024-11-25T12:45:14,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741874_1051 (size=3256) 2024-11-25T12:45:15,221 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139, size=3.2 K (3256bytes) 2024-11-25T12:45:15,221 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 2024-11-25T12:45:15,222 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 after 1ms 2024-11-25T12:45:15,250 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:15,251 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 took 29ms 2024-11-25T12:45:15,254 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 so closing down 2024-11-25T12:45:15,254 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:15,256 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538714139.temp 2024-11-25T12:45:15,284 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp 2024-11-25T12:45:15,284 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:15,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741877_1054 (size=2650) 2024-11-25T12:45:15,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741877_1054 (size=2650) 2024-11-25T12:45:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741877_1054 (size=2650) 2024-11-25T12:45:15,294 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:15,296 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp to hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000020 2024-11-25T12:45:15,296 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-25T12:45:15,296 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139, journal: Splitting hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139, size=3.2 K (3256bytes) at 1732538715221Finishing writing output for hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139 so closing down at 1732538715254 (+33 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp at 1732538715284 (+30 ms)3 split writer threads finished at 1732538715284Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732538715294 (+10 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000003-wal.1732538714139.temp to hdfs://localhost:41185/hbase/data/default/testSequentialEditLogSeqNum/a1a6b17902032562890f818c81af74ba/recovered.edits/0000000000000000020 at 1732538715296 (+2 ms)Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732538714131/wal.1732538714139, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732538715296 2024-11-25T12:45:15,310 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:54786 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 546), ProcessCount=11 (was 11), AvailableMemoryMB=2062 (was 2138) 2024-11-25T12:45:15,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=546, ProcessCount=11, AvailableMemoryMB=2061 2024-11-25T12:45:15,337 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:15,340 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:15,364 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:15,366 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50026384, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-50026384, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:15,380 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50026384/hregion-50026384.1732538715367, exclude list is [], retry=0 2024-11-25T12:45:15,383 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:15,383 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:15,383 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:15,423 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50026384/hregion-50026384.1732538715367 2024-11-25T12:45:15,424 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:15,424 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 808cc59dd64dc853983f14a68dc96dd0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:15,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:45:15,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:45:15,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:15,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741879_1056 (size=70) 2024-11-25T12:45:15,442 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,443 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName a 2024-11-25T12:45:15,444 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,444 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,445 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,447 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName b 2024-11-25T12:45:15,447 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,448 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,448 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,450 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName c 2024-11-25T12:45:15,450 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,451 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,451 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,453 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,453 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,454 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,454 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,455 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:15,456 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,459 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:15,460 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 808cc59dd64dc853983f14a68dc96dd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60461104, jitterRate=-0.0990593433380127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:15,461 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 808cc59dd64dc853983f14a68dc96dd0: Writing region info on filesystem at 1732538715440Initializing all the Stores at 1732538715441 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715441Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715441Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715441Cleaning up temporary data from old regions at 1732538715454 (+13 ms)Region opened successfully at 1732538715461 (+7 ms) 2024-11-25T12:45:15,461 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 808cc59dd64dc853983f14a68dc96dd0, disabling compactions & flushes 2024-11-25T12:45:15,461 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:15,461 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:15,461 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. after waiting 0 ms 2024-11-25T12:45:15,461 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:15,462 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:15,462 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 808cc59dd64dc853983f14a68dc96dd0: Waiting for close lock at 1732538715461Disabling compacts and flushes for region at 1732538715461Disabling writes for close at 1732538715461Writing region close event to WAL at 1732538715462 (+1 ms)Closed at 1732538715462 2024-11-25T12:45:15,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741878_1055 (size=175) 2024-11-25T12:45:15,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741878_1055 (size=175) 2024-11-25T12:45:15,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741878_1055 (size=175) 2024-11-25T12:45:15,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:15,475 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50026384:(num 1732538715367) 2024-11-25T12:45:15,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:15,478 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:15,497 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, exclude list is [], retry=0 2024-11-25T12:45:15,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:15,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:15,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:15,539 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 2024-11-25T12:45:15,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:15,540 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 808cc59dd64dc853983f14a68dc96dd0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:15,540 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:15,541 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,541 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,547 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,550 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName a 2024-11-25T12:45:15,550 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,550 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,551 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,552 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName b 2024-11-25T12:45:15,552 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,553 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,553 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,554 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName c 2024-11-25T12:45:15,555 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:15,555 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:15,556 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,559 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,559 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,560 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:15,567 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:15,568 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 808cc59dd64dc853983f14a68dc96dd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62117751, jitterRate=-0.07437337934970856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:15,569 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 808cc59dd64dc853983f14a68dc96dd0: Writing region info on filesystem at 1732538715541Initializing all the Stores at 1732538715543 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715547 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538715547Cleaning up temporary data from old regions at 1732538715559 (+12 ms)Region opened successfully at 1732538715568 (+9 ms) 2024-11-25T12:45:15,573 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732538715572/Put/seqid=0 2024-11-25T12:45:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:45:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:45:15,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741881_1058 (size=4826) 2024-11-25T12:45:15,592 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41185/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 808cc59dd64dc853983f14a68dc96dd0/a 2024-11-25T12:45:15,604 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-25T12:45:15,605 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:45:15,605 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 808cc59dd64dc853983f14a68dc96dd0: 2024-11-25T12:45:15,608 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/012efc6986e94933a9936422365561bc_SeqId_3_ 2024-11-25T12:45:15,609 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41185/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 808cc59dd64dc853983f14a68dc96dd0/a as hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/012efc6986e94933a9936422365561bc_SeqId_3_ - updating store file list. 2024-11-25T12:45:15,620 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 012efc6986e94933a9936422365561bc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:15,621 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/012efc6986e94933a9936422365561bc_SeqId_3_ into 808cc59dd64dc853983f14a68dc96dd0/a 2024-11-25T12:45:15,621 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41185/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 808cc59dd64dc853983f14a68dc96dd0/a (new location: hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/012efc6986e94933a9936422365561bc_SeqId_3_) 2024-11-25T12:45:15,683 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, size=0 (0bytes) 2024-11-25T12:45:15,683 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 might be still open, length is 0 2024-11-25T12:45:15,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 2024-11-25T12:45:15,684 WARN [IPC Server handler 3 on default port 41185 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-25T12:45:15,684 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 after 1ms 2024-11-25T12:45:15,984 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53964 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53964 dst: /127.0.0.1:38775 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38775 remote=/127.0.0.1:53964]. Total timeout mills is 60000, 59646 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:15,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:45902 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35695:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45902 dst: /127.0.0.1:35695 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:15,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:48600 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45631:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48600 dst: /127.0.0.1:45631 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:16,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741880_1059 (size=606) 2024-11-25T12:45:19,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-25T12:45:19,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:19,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-25T12:45:19,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:19,685 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 after 4002ms 2024-11-25T12:45:19,719 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:19,719 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 took 4036ms 2024-11-25T12:45:19,722 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478; continuing. 2024-11-25T12:45:19,722 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 so closing down 2024-11-25T12:45:19,722 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:19,724 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732538715478.temp 2024-11-25T12:45:19,770 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp 2024-11-25T12:45:19,770 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741882_1060 (size=329) 2024-11-25T12:45:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741882_1060 (size=329) 2024-11-25T12:45:19,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741882_1060 (size=329) 2024-11-25T12:45:19,788 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:19,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp to hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 2024-11-25T12:45:19,790 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 70 ms; skipped=1; WAL=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:45:19,790 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, journal: Splitting hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, size=0 (0bytes) at 1732538715683Finishing writing output for hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 so closing down at 1732538719722 (+4039 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp at 1732538719770 (+48 ms)3 split writer threads finished at 1732538719770Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732538719788 (+18 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005-wal.1732538715478.temp to hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 at 1732538719790 (+2 ms)Processed 2 edits across 1 Regions in 70 ms; skipped=1; WAL=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478, size=0, length=0, corrupted=false, cancelled=false at 1732538719790 2024-11-25T12:45:19,793 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538715478 2024-11-25T12:45:19,795 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 2024-11-25T12:45:19,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:19,798 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:19,812 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538719799, exclude list is [], retry=0 2024-11-25T12:45:19,816 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:19,816 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:19,817 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:19,866 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538719799 2024-11-25T12:45:19,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:19,867 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 808cc59dd64dc853983f14a68dc96dd0, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:19,868 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:19,868 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,868 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,876 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,878 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName a 2024-11-25T12:45:19,878 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:19,904 DEBUG [StoreFileOpener-808cc59dd64dc853983f14a68dc96dd0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 012efc6986e94933a9936422365561bc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:19,904 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/012efc6986e94933a9936422365561bc_SeqId_3_ 2024-11-25T12:45:19,905 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:19,905 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,907 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName b 2024-11-25T12:45:19,907 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:19,908 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:19,908 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,909 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 808cc59dd64dc853983f14a68dc96dd0 columnFamilyName c 2024-11-25T12:45:19,909 DEBUG [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:19,910 INFO [StoreOpener-808cc59dd64dc853983f14a68dc96dd0-1 {}] regionserver.HStore(327): Store=808cc59dd64dc853983f14a68dc96dd0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:19,910 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:19,916 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 2024-11-25T12:45:19,953 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:19,955 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 2024-11-25T12:45:19,955 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 808cc59dd64dc853983f14a68dc96dd0 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-25T12:45:19,976 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/.tmp/a/5cb14a6cec8345a5bd7eea16de11d8fe is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732538715629/Put/seqid=0 2024-11-25T12:45:19,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:45:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:45:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741884_1062 (size=5149) 2024-11-25T12:45:19,998 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/.tmp/a/5cb14a6cec8345a5bd7eea16de11d8fe 2024-11-25T12:45:20,015 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/.tmp/a/5cb14a6cec8345a5bd7eea16de11d8fe as hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/5cb14a6cec8345a5bd7eea16de11d8fe 2024-11-25T12:45:20,026 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/a/5cb14a6cec8345a5bd7eea16de11d8fe, entries=1, sequenceid=5, filesize=5.0 K 2024-11-25T12:45:20,026 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 808cc59dd64dc853983f14a68dc96dd0 in 71ms, sequenceid=5, compaction requested=false; wal=null 2024-11-25T12:45:20,027 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/0000000000000000005 2024-11-25T12:45:20,029 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:20,029 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:20,030 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:20,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 808cc59dd64dc853983f14a68dc96dd0 2024-11-25T12:45:20,043 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/808cc59dd64dc853983f14a68dc96dd0/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T12:45:20,045 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 808cc59dd64dc853983f14a68dc96dd0; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66886111, jitterRate=-0.003319278359413147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:20,045 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 808cc59dd64dc853983f14a68dc96dd0: Writing region info on filesystem at 1732538719868Initializing all the Stores at 1732538719870 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538719871 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538719875 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538719875Obtaining lock to block concurrent updates at 1732538719955 (+80 ms)Preparing flush snapshotting stores in 808cc59dd64dc853983f14a68dc96dd0 at 1732538719955Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732538719955Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. at 1732538719955Flushing 808cc59dd64dc853983f14a68dc96dd0/a: creating writer at 1732538719955Flushing 808cc59dd64dc853983f14a68dc96dd0/a: appending metadata at 1732538719976 (+21 ms)Flushing 808cc59dd64dc853983f14a68dc96dd0/a: closing flushed file at 1732538719976Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@642de661: reopening flushed file at 1732538720013 (+37 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 808cc59dd64dc853983f14a68dc96dd0 in 71ms, sequenceid=5, compaction requested=false; wal=null at 1732538720026 (+13 ms)Cleaning up temporary data from old regions at 1732538720029 (+3 ms)Region opened successfully at 1732538720045 (+16 ms) 2024-11-25T12:45:20,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 808cc59dd64dc853983f14a68dc96dd0, disabling compactions & flushes 2024-11-25T12:45:20,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:20,052 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:20,052 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. after waiting 0 ms 2024-11-25T12:45:20,052 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:20,053 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732538715338.808cc59dd64dc853983f14a68dc96dd0. 2024-11-25T12:45:20,054 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 808cc59dd64dc853983f14a68dc96dd0: Waiting for close lock at 1732538720052Disabling compacts and flushes for region at 1732538720052Disabling writes for close at 1732538720052Writing region close event to WAL at 1732538720053 (+1 ms)Closed at 1732538720053 2024-11-25T12:45:20,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741883_1061 (size=175) 2024-11-25T12:45:20,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741883_1061 (size=175) 2024-11-25T12:45:20,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741883_1061 (size=175) 2024-11-25T12:45:20,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:20,078 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538719799) 2024-11-25T12:45:20,105 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1499292629_22 at /127.0.0.1:45926 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41185 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1499292629_22 at /127.0.0.1:53992 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=646 (was 546) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1897 (was 2061) 2024-11-25T12:45:20,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=646, ProcessCount=11, AvailableMemoryMB=1895 2024-11-25T12:45:20,151 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:20,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:45:20,164 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ff8009e463e,43183,1732538699066 2024-11-25T12:45:20,167 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16eebb1e 2024-11-25T12:45:20,170 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:45:20,176 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:45:20,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:45:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T12:45:20,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:45:20,205 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:20,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-25T12:45:20,208 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:45:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:45:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:45:20,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:45:20,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741885_1063 (size=694) 2024-11-25T12:45:20,252 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3 2024-11-25T12:45:20,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:45:20,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:45:20,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741886_1064 (size=77) 2024-11-25T12:45:20,300 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:20,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:20,301 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:20,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,301 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,301 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538720301Disabling compacts and flushes for region at 1732538720301Disabling writes for close at 1732538720301Writing region close event to WAL at 1732538720301Closed at 1732538720301 2024-11-25T12:45:20,306 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:45:20,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732538720306"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732538720306"}]},"ts":"1732538720306"} 2024-11-25T12:45:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:45:20,328 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:45:20,334 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:45:20,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732538720334"}]},"ts":"1732538720334"} 2024-11-25T12:45:20,343 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-25T12:45:20,344 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:45:20,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T12:45:20,348 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:45:20,348 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:45:20,349 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T12:45:20,349 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:45:20,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN}] 2024-11-25T12:45:20,358 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN 2024-11-25T12:45:20,365 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN; state=OFFLINE, location=1ff8009e463e,37815,1732538700283; forceNewPlan=false, retain=false 2024-11-25T12:45:20,518 INFO [1ff8009e463e:43183 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:45:20,519 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPENING, regionLocation=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:20,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN because future has completed 2024-11-25T12:45:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:45:20,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283}] 2024-11-25T12:45:20,691 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:45:20,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:45:20,721 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,722 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:20,723 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,723 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:20,723 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,724 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,727 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,730 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:20,730 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:20,735 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:20,736 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,738 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:20,738 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:20,739 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:20,740 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,741 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,742 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,743 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,745 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:45:20,748 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,752 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:20,753 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 88bc9a72f668c659e38c0669352cfc0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65114299, jitterRate=-0.02972133457660675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:45:20,753 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,754 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 88bc9a72f668c659e38c0669352cfc0c: Running coprocessor pre-open hook at 1732538720724Writing region info on filesystem at 1732538720724Initializing all the Stores at 1732538720726 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538720726Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538720727 (+1 ms)Cleaning up temporary data from old regions at 1732538720743 (+16 ms)Running coprocessor post-open hooks at 1732538720753 (+10 ms)Region opened successfully at 1732538720754 (+1 ms) 2024-11-25T12:45:20,756 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., pid=6, masterSystemTime=1732538720691 2024-11-25T12:45:20,761 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPEN, openSeqNum=2, regionLocation=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:20,761 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,761 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 because future has completed 2024-11-25T12:45:20,771 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43183 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1ff8009e463e,37815,1732538700283, table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T12:45:20,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T12:45:20,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 in 240 msec 2024-11-25T12:45:20,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T12:45:20,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN in 430 msec 2024-11-25T12:45:20,788 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:45:20,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732538720788"}]},"ts":"1732538720788"} 2024-11-25T12:45:20,791 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-25T12:45:20,793 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:45:20,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 609 msec 2024-11-25T12:45:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:45:20,849 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-25T12:45:20,849 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-25T12:45:20,851 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T12:45:20,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-25T12:45:20,862 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T12:45:20,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-25T12:45:20,883 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=2] 2024-11-25T12:45:20,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:45:20,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:45:20,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=88bc9a72f668c659e38c0669352cfc0c, source=1ff8009e463e,37815,1732538700283, destination=1ff8009e463e,35011,1732538700146, warming up region on 1ff8009e463e,35011,1732538700146 2024-11-25T12:45:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:45:20,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=88bc9a72f668c659e38c0669352cfc0c, source=1ff8009e463e,37815,1732538700283, destination=1ff8009e463e,35011,1732538700146, running balancer 2024-11-25T12:45:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE 2024-11-25T12:45:20,935 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE 2024-11-25T12:45:20,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44349, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:45:20,938 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=CLOSING, regionLocation=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:20,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE because future has completed 2024-11-25T12:45:20,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(7855): Warmup {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:20,946 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:45:20,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283}] 2024-11-25T12:45:20,951 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,958 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:20,959 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:20,961 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:20,962 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:20,964 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:20,964 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:20,965 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:20,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538720965Disabling compacts and flushes for region at 1732538720965Disabling writes for close at 1732538720965Writing region close event to WAL at 1732538720966 (+1 ms)Closed at 1732538720966 2024-11-25T12:45:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T12:45:21,118 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,119 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:45:21,120 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:21,120 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,120 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,120 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:21,120 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,120 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 88bc9a72f668c659e38c0669352cfc0c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-25T12:45:21,148 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/11513c2c62444268ab70d3c3f1bcc67e is 35, key is r1/cf1:q/1732538720905/Put/seqid=0 2024-11-25T12:45:21,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:45:21,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:45:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741887_1065 (size=4783) 2024-11-25T12:45:21,572 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/11513c2c62444268ab70d3c3f1bcc67e 2024-11-25T12:45:21,582 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/11513c2c62444268ab70d3c3f1bcc67e as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e 2024-11-25T12:45:21,590 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e, entries=1, sequenceid=5, filesize=4.7 K 2024-11-25T12:45:21,591 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 88bc9a72f668c659e38c0669352cfc0c in 471ms, sequenceid=5, compaction requested=false 2024-11-25T12:45:21,591 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-25T12:45:21,598 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T12:45:21,601 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,601 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538721119Running coprocessor pre-close hooks at 1732538721119Disabling compacts and flushes for region at 1732538721120 (+1 ms)Disabling writes for close at 1732538721120Obtaining lock to block concurrent updates at 1732538721120Preparing flush snapshotting stores in 88bc9a72f668c659e38c0669352cfc0c at 1732538721120Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732538721121 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. at 1732538721128 (+7 ms)Flushing 88bc9a72f668c659e38c0669352cfc0c/cf1: creating writer at 1732538721128Flushing 88bc9a72f668c659e38c0669352cfc0c/cf1: appending metadata at 1732538721147 (+19 ms)Flushing 88bc9a72f668c659e38c0669352cfc0c/cf1: closing flushed file at 1732538721148 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4755779f: reopening flushed file at 1732538721581 (+433 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 88bc9a72f668c659e38c0669352cfc0c in 471ms, sequenceid=5, compaction requested=false at 1732538721591 (+10 ms)Writing region close event to WAL at 1732538721592 (+1 ms)Running coprocessor post-close hooks at 1732538721598 (+6 ms)Closed at 1732538721601 (+3 ms) 2024-11-25T12:45:21,601 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 88bc9a72f668c659e38c0669352cfc0c move to 1ff8009e463e,35011,1732538700146 record at close sequenceid=5 2024-11-25T12:45:21,604 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,605 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=CLOSED 2024-11-25T12:45:21,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 because future has completed 2024-11-25T12:45:21,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T12:45:21,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 in 663 msec 2024-11-25T12:45:21,617 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,35011,1732538700146; forceNewPlan=false, retain=false 2024-11-25T12:45:21,768 INFO [1ff8009e463e:43183 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:45:21,769 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPENING, regionLocation=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:21,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE because future has completed 2024-11-25T12:45:21,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146}] 2024-11-25T12:45:21,936 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,936 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:21,936 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,936 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:21,937 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,937 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,941 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,942 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:21,942 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:21,952 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e 2024-11-25T12:45:21,952 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:21,952 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,953 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:21,954 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:21,954 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:21,954 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,956 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,957 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,958 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,958 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,959 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:45:21,961 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,961 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 88bc9a72f668c659e38c0669352cfc0c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59331332, jitterRate=-0.11589425802230835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:45:21,962 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:21,962 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 88bc9a72f668c659e38c0669352cfc0c: Running coprocessor pre-open hook at 1732538721937Writing region info on filesystem at 1732538721937Initializing all the Stores at 1732538721938 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538721938Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538721940 (+2 ms)Cleaning up temporary data from old regions at 1732538721958 (+18 ms)Running coprocessor post-open hooks at 1732538721962 (+4 ms)Region opened successfully at 1732538721962 2024-11-25T12:45:21,968 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., pid=9, masterSystemTime=1732538721931 2024-11-25T12:45:21,975 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,975 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:21,975 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPEN, openSeqNum=9, regionLocation=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:21,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146 because future has completed 2024-11-25T12:45:22,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T12:45:22,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146 in 220 msec 2024-11-25T12:45:22,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE in 1.0760 sec 2024-11-25T12:45:22,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:45:22,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:45:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:56652 deadline: 1732538782070, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=35011 startCode=1732538700146. As of locationSeqNum=5. 2024-11-25T12:45:22,080 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=35011 startCode=1732538700146. As of locationSeqNum=5. 2024-11-25T12:45:22,081 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=35011 startCode=1732538700146. As of locationSeqNum=5. 2024-11-25T12:45:22,081 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,35011,1732538700146, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=35011 startCode=1732538700146. As of locationSeqNum=5. 2024-11-25T12:45:22,202 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:45:22,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:45:22,232 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88bc9a72f668c659e38c0669352cfc0c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-25T12:45:22,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/3176e2c15c5e48de93d06d78483e151e is 29, key is r1/cf1:/1732538722208/DeleteFamily/seqid=0 2024-11-25T12:45:22,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:45:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:45:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741888_1066 (size=4906) 2024-11-25T12:45:22,300 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,309 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,333 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf2/ecea748af58b487390f395ce81245d35 is 29, key is r1/cf2:/1732538722208/DeleteFamily/seqid=0 2024-11-25T12:45:22,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:45:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:45:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741889_1067 (size=4906) 2024-11-25T12:45:22,376 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf2/ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,383 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/3176e2c15c5e48de93d06d78483e151e as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,393 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,394 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T12:45:22,395 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf2/ecea748af58b487390f395ce81245d35 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,406 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,406 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T12:45:22,408 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 88bc9a72f668c659e38c0669352cfc0c in 175ms, sequenceid=12, compaction requested=false 2024-11-25T12:45:22,408 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 88bc9a72f668c659e38c0669352cfc0c: 2024-11-25T12:45:22,411 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:45:22,413 DEBUG [Time-limited test {}] regionserver.HStore(1541): 88bc9a72f668c659e38c0669352cfc0c/cf1 is initiating major compaction (all files) 2024-11-25T12:45:22,413 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:22,413 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:22,414 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 88bc9a72f668c659e38c0669352cfc0c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,414 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e, hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e] into tmpdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp, totalSize=9.5 K 2024-11-25T12:45:22,415 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 11513c2c62444268ab70d3c3f1bcc67e, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732538720905 2024-11-25T12:45:22,416 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3176e2c15c5e48de93d06d78483e151e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T12:45:22,430 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 88bc9a72f668c659e38c0669352cfc0c#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:45:22,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:45:22,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:45:22,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741890_1068 (size=4626) 2024-11-25T12:45:22,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf1/44aa06f2598146c7932d220739e9a95d as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/44aa06f2598146c7932d220739e9a95d 2024-11-25T12:45:22,488 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 88bc9a72f668c659e38c0669352cfc0c/cf1 of 88bc9a72f668c659e38c0669352cfc0c into 44aa06f2598146c7932d220739e9a95d(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:45:22,488 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 88bc9a72f668c659e38c0669352cfc0c: 2024-11-25T12:45:22,489 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T12:45:22,489 DEBUG [Time-limited test {}] regionserver.HStore(1541): 88bc9a72f668c659e38c0669352cfc0c/cf2 is initiating major compaction (all files) 2024-11-25T12:45:22,490 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:45:22,490 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:45:22,490 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 88bc9a72f668c659e38c0669352cfc0c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,490 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35] into tmpdir=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp, totalSize=4.8 K 2024-11-25T12:45:22,491 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ecea748af58b487390f395ce81245d35, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T12:45:22,505 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 88bc9a72f668c659e38c0669352cfc0c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:45:22,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:45:22,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:45:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741891_1069 (size=4592) 2024-11-25T12:45:22,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/.tmp/cf2/66b173e23ae744eab0f965fe130ec320 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/66b173e23ae744eab0f965fe130ec320 2024-11-25T12:45:22,553 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 88bc9a72f668c659e38c0669352cfc0c/cf2 of 88bc9a72f668c659e38c0669352cfc0c into 66b173e23ae744eab0f965fe130ec320(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:45:22,554 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 88bc9a72f668c659e38c0669352cfc0c: 2024-11-25T12:45:22,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=88bc9a72f668c659e38c0669352cfc0c, source=1ff8009e463e,35011,1732538700146, destination=1ff8009e463e,37815,1732538700283, warming up region on 1ff8009e463e,37815,1732538700283 2024-11-25T12:45:22,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=88bc9a72f668c659e38c0669352cfc0c, source=1ff8009e463e,35011,1732538700146, destination=1ff8009e463e,37815,1732538700283, running balancer 2024-11-25T12:45:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE 2024-11-25T12:45:22,561 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE 2024-11-25T12:45:22,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=CLOSING, regionLocation=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:22,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(7855): Warmup {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:22,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE because future has completed 2024-11-25T12:45:22,568 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:22,568 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:45:22,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146}] 2024-11-25T12:45:22,569 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:22,569 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:22,582 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e 2024-11-25T12:45:22,591 INFO [StoreFileOpener-88bc9a72f668c659e38c0669352cfc0c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,592 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,603 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/44aa06f2598146c7932d220739e9a95d 2024-11-25T12:45:22,603 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:22,603 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:22,605 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:22,605 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-25T12:45:22,617 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/66b173e23ae744eab0f965fe130ec320 2024-11-25T12:45:22,638 INFO [StoreFileOpener-88bc9a72f668c659e38c0669352cfc0c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,638 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,638 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:22,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538722638Disabling compacts and flushes for region at 1732538722638Disabling writes for close at 1732538722639 (+1 ms)Writing region close event to WAL at 1732538722645 (+6 ms)Closed at 1732538722645 2024-11-25T12:45:22,723 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:22,723 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:45:22,724 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:22,724 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,724 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,724 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:22,724 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,726 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e, hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e] to archive 2024-11-25T12:45:22,730 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:45:22,734 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/11513c2c62444268ab70d3c3f1bcc67e 2024-11-25T12:45:22,737 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/3176e2c15c5e48de93d06d78483e151e 2024-11-25T12:45:22,752 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35] to archive 2024-11-25T12:45:22,753 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:45:22,755 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35 to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/ecea748af58b487390f395ce81245d35 2024-11-25T12:45:22,762 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-25T12:45:22,770 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:22,770 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538722724Running coprocessor pre-close hooks at 1732538722724Disabling compacts and flushes for region at 1732538722724Disabling writes for close at 1732538722724Writing region close event to WAL at 1732538722757 (+33 ms)Running coprocessor post-close hooks at 1732538722770 (+13 ms)Closed at 1732538722770 2024-11-25T12:45:22,770 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 88bc9a72f668c659e38c0669352cfc0c move to 1ff8009e463e,37815,1732538700283 record at close sequenceid=12 2024-11-25T12:45:22,774 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:22,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=CLOSED 2024-11-25T12:45:22,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146 because future has completed 2024-11-25T12:45:22,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T12:45:22,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,35011,1732538700146 in 213 msec 2024-11-25T12:45:22,796 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE; state=CLOSED, location=1ff8009e463e,37815,1732538700283; forceNewPlan=false, retain=false 2024-11-25T12:45:22,947 INFO [1ff8009e463e:43183 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T12:45:22,948 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPENING, regionLocation=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:22,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE because future has completed 2024-11-25T12:45:22,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283}] 2024-11-25T12:45:23,118 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,119 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:23,119 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,119 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:23,119 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,119 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,121 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,122 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:23,122 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:23,130 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/44aa06f2598146c7932d220739e9a95d 2024-11-25T12:45:23,131 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:23,131 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,132 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:23,132 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:23,142 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/66b173e23ae744eab0f965fe130ec320 2024-11-25T12:45:23,142 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:23,143 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,144 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,146 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,147 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,147 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,148 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:45:23,149 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,151 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 88bc9a72f668c659e38c0669352cfc0c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59102717, jitterRate=-0.11930088698863983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:45:23,151 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,152 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 88bc9a72f668c659e38c0669352cfc0c: Running coprocessor pre-open hook at 1732538723119Writing region info on filesystem at 1732538723119Initializing all the Stores at 1732538723121 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538723121Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538723121Cleaning up temporary data from old regions at 1732538723147 (+26 ms)Running coprocessor post-open hooks at 1732538723151 (+4 ms)Region opened successfully at 1732538723152 (+1 ms) 2024-11-25T12:45:23,153 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., pid=12, masterSystemTime=1732538723111 2024-11-25T12:45:23,158 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,158 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPEN, openSeqNum=18, regionLocation=1ff8009e463e,37815,1732538700283 2024-11-25T12:45:23,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 because future has completed 2024-11-25T12:45:23,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-25T12:45:23,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,37815,1732538700283 in 207 msec 2024-11-25T12:45:23,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, REOPEN/MOVE in 609 msec 2024-11-25T12:45:23,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:45:23,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:45:23,222 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1ff8009e463e,37815,1732538700283: testing ***** 2024-11-25T12:45:23,222 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-25T12:45:23,225 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-25T12:45:23,227 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-25T12:45:23,234 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-25T12:45:23,248 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-25T12:45:23,266 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 496138120 }, "NonHeapMemoryUsage": { "committed": 172556288, "init": 7667712, "max": -1, "used": 170014472 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 85, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 85, "ProcessCallTime_99th_percentile": 85, "ProcessCallTime_99.9th_percentile": 85, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 9, "ProcessCallTime_TimeRangeCount_10-30": 2, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 6, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 6, "QueueCallTime_99th_percentile": 6, "QueueCallTime_99.9th_percentile": 6, "QueueCallTime_TimeRangeCount_0-1": 30, "QueueCallTime_TimeRangeCount_3-10": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 85, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 85, "TotalCallTime_99th_percentile": 85, "TotalCallTime_99.9th_percentile": 85, "TotalCallTime_TimeRangeCount_0-1": 7, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 9, "TotalCallTime_TimeRangeCount_10-30": 3, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5753 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59707", "tag.serverName": "1ff8009e463e,37815,1732538700283", "tag.clusterId": "0c41da9c-186e-443b-b8c3-e118f36dec4e", "tag.Context": "regionserver", "tag.Hostname": "1ff8009e463e", "regionCount": 1, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732538700283, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 3, "Put_max": 3, "Put_mean": 3, "Put_25th_percentile": 3, "Put_median": 3, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 471, "FlushTime_max": 471, "FlushTime_mean": 471, "FlushTime_25th_percentile": 471, "FlushTime_median": 471, "FlushTime_75th_percentile": 471, "FlushTime_90th_percentile": 471, "FlushTime_95th_percentile": 471, "FlushTime_98th_percentile": 471, "FlushTime_99th_percentile": 471, "FlushTime_99.9th_percentile": 471, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-25T12:45:23,270 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43183 {}] master.MasterRpcServices(700): 1ff8009e463e,37815,1732538700283 reported a fatal error: ***** ABORTING region server 1ff8009e463e,37815,1732538700283: testing ***** 2024-11-25T12:45:23,287 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,37815,1732538700283' ***** 2024-11-25T12:45:23,288 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-25T12:45:23,288 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:45:23,288 INFO [RS:1;1ff8009e463e:37815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-25T12:45:23,288 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:45:23,289 INFO [RS:1;1ff8009e463e:37815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-25T12:45:23,289 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(3091): Received CLOSE for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,289 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(956): aborting server 1ff8009e463e,37815,1732538700283 2024-11-25T12:45:23,289 INFO [RS:1;1ff8009e463e:37815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:45:23,290 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:23,290 INFO [RS:1;1ff8009e463e:37815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1ff8009e463e:37815. 2024-11-25T12:45:23,290 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,290 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,290 DEBUG [RS:1;1ff8009e463e:37815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:45:23,290 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:23,290 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,290 DEBUG [RS:1;1ff8009e463e:37815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:23,292 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T12:45:23,293 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1325): Online Regions={88bc9a72f668c659e38c0669352cfc0c=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.} 2024-11-25T12:45:23,293 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:23,293 DEBUG [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1351): Waiting on 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35011 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:33930 deadline: 1732538783295, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=37815 startCode=1732538700283. As of locationSeqNum=12. 2024-11-25T12:45:23,296 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,35011,1732538700146, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,35011,1732538700146, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=37815 startCode=1732538700283. As of locationSeqNum=12. 2024-11-25T12:45:23,296 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,35011,1732538700146, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=37815 startCode=1732538700283. As of locationSeqNum=12. 2024-11-25T12:45:23,296 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,35011,1732538700146, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ff8009e463e port=37815 startCode=1732538700283. As of locationSeqNum=12. 2024-11-25T12:45:23,303 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,303 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538723289Running coprocessor pre-close hooks at 1732538723289Disabling compacts and flushes for region at 1732538723289Disabling writes for close at 1732538723290 (+1 ms)Writing region close event to WAL at 1732538723303 (+13 ms)Running coprocessor post-close hooks at 1732538723303Closed at 1732538723303 2024-11-25T12:45:23,304 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:23,317 INFO [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T12:45:23,317 INFO [regionserver/1ff8009e463e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T12:45:23,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,37815,1732538700283 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:23,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,37815,1732538700283 aborting 2024-11-25T12:45:23,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ff8009e463e,37815,1732538700283 aborting 2024-11-25T12:45:23,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=12 from cache 2024-11-25T12:45:23,493 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,37815,1732538700283; all regions closed. 2024-11-25T12:45:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741834_1010 (size=2288) 2024-11-25T12:45:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741834_1010 (size=2288) 2024-11-25T12:45:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741834_1010 (size=2288) 2024-11-25T12:45:23,498 DEBUG [RS:1;1ff8009e463e:37815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:23,498 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:23,499 INFO [RS:1;1ff8009e463e:37815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:45:23,499 INFO [RS:1;1ff8009e463e:37815 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T12:45:23,499 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:45:23,499 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:45:23,499 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:45:23,500 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:45:23,500 INFO [RS:1;1ff8009e463e:37815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:45:23,500 INFO [RS:1;1ff8009e463e:37815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37815 2024-11-25T12:45:23,546 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,37815,1732538700283 2024-11-25T12:45:23,546 INFO [RS:1;1ff8009e463e:37815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:45:23,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:23,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,37815,1732538700283] 2024-11-25T12:45:23,565 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,37815,1732538700283 already deleted, retry=false 2024-11-25T12:45:23,565 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1ff8009e463e,37815,1732538700283 on 1ff8009e463e,43183,1732538699066 2024-11-25T12:45:23,570 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1ff8009e463e,37815,1732538700283, splitWal=true, meta=false 2024-11-25T12:45:23,572 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 1ff8009e463e,37815,1732538700283 (carryingMeta=false) 1ff8009e463e,37815,1732538700283/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1ee4c81d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T12:45:23,573 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1ff8009e463e,37815,1732538700283, splitWal=true, meta=false 2024-11-25T12:45:23,574 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1ff8009e463e,37815,1732538700283 had 1 regions 2024-11-25T12:45:23,576 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1ff8009e463e,37815,1732538700283, splitWal=true, meta=false, isMeta: false 2024-11-25T12:45:23,577 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting 2024-11-25T12:45:23,578 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ff8009e463e,37815,1732538700283 WAL count=1, meta=false 2024-11-25T12:45:23,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ff8009e463e%2C37815%2C1732538700283.1732538702495}] 2024-11-25T12:45:23,587 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:23,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C37815%2C1732538700283.1732538702495, worker=1ff8009e463e,35011,1732538700146}] 2024-11-25T12:45:23,624 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18] 2024-11-25T12:45:23,626 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ff8009e463e:37815 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:37815 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:23,627 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18, error=java.net.ConnectException: Call to address=1ff8009e463e:37815 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:37815 2024-11-25T12:45:23,628 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-25T12:45:23,628 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 from cache 2024-11-25T12:45:23,628 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 1ff8009e463e:37815 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ff8009e463e/172.17.0.3:37815 2024-11-25T12:45:23,658 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:23,658 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10173503b590002, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:23,658 INFO [RS:1;1ff8009e463e:37815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:45:23,658 INFO [RS:1;1ff8009e463e:37815 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,37815,1732538700283; zookeeper connection closed. 2024-11-25T12:45:23,659 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@770978fe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@770978fe 2024-11-25T12:45:23,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35011 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-25T12:45:23,770 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495, size=2.2 K (2288bytes) 2024-11-25T12:45:23,770 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 2024-11-25T12:45:23,771 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 after 1ms 2024-11-25T12:45:23,810 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:23,810 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 took 40ms 2024-11-25T12:45:23,819 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 88bc9a72f668c659e38c0669352cfc0c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-25T12:45:23,819 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 so closing down 2024-11-25T12:45:23,819 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:23,819 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:23,819 INFO [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-25T12:45:23,820 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495, journal: Splitting hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495, size=2.2 K (2288bytes) at 1732538723770Finishing writing output for hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 so closing down at 1732538723819 (+49 ms)3 split writer threads finished at 1732538723819Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732538723820 (+1 ms) 2024-11-25T12:45:23,820 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 2024-11-25T12:45:23,821 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ff8009e463e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-25T12:45:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43183 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-25T12:45:23,830 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting/1ff8009e463e%2C37815%2C1732538700283.1732538702495 to hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs 2024-11-25T12:45:23,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T12:45:23,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ff8009e463e%2C37815%2C1732538700283.1732538702495, worker=1ff8009e463e,35011,1732538700146 in 241 msec 2024-11-25T12:45:23,836 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=1ff8009e463e,35011,1732538700146 2024-11-25T12:45:23,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T12:45:23,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ff8009e463e%2C37815%2C1732538700283.1732538702495, worker=1ff8009e463e,35011,1732538700146 in 257 msec 2024-11-25T12:45:23,843 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting dir is empty, no logs to split. 2024-11-25T12:45:23,843 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1ff8009e463e,37815,1732538700283 WAL count=0, meta=false 2024-11-25T12:45:23,843 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1ff8009e463e,37815,1732538700283 WAL splitting is done? wals=0, meta=false 2024-11-25T12:45:23,846 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 1ff8009e463e,37815,1732538700283 failed, ignore...File hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/WALs/1ff8009e463e,37815,1732538700283-splitting does not exist. 2024-11-25T12:45:23,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN}] 2024-11-25T12:45:23,851 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN 2024-11-25T12:45:23,852 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T12:45:23,934 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18] 2024-11-25T12:45:23,935 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ff8009e463e:37815 this server is in the failed servers list 2024-11-25T12:45:23,935 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ff8009e463e:37815 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:37815 2024-11-25T12:45:23,936 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ff8009e463e:37815 2024-11-25T12:45:23,936 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,37815,1732538700283, seqNum=18 from cache 2024-11-25T12:45:24,003 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(204): Hosts are {1ff8009e463e=0} racks are {/default-rack=0} 2024-11-25T12:45:24,003 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T12:45:24,003 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T12:45:24,003 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T12:45:24,003 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T12:45:24,004 INFO [1ff8009e463e:43183 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T12:45:24,004 INFO [1ff8009e463e:43183 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T12:45:24,004 DEBUG [1ff8009e463e:43183 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T12:45:24,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPENING, regionLocation=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:24,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN because future has completed 2024-11-25T12:45:24,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,44737,1732538700372}] 2024-11-25T12:45:24,171 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:24,172 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bc9a72f668c659e38c0669352cfc0c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:24,172 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,172 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:24,172 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,172 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,175 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,177 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf1 2024-11-25T12:45:24,177 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,186 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf1/44aa06f2598146c7932d220739e9a95d 2024-11-25T12:45:24,186 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,187 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,188 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc9a72f668c659e38c0669352cfc0c columnFamilyName cf2 2024-11-25T12:45:24,188 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,197 DEBUG [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/cf2/66b173e23ae744eab0f965fe130ec320 2024-11-25T12:45:24,197 INFO [StoreOpener-88bc9a72f668c659e38c0669352cfc0c-1 {}] regionserver.HStore(327): Store=88bc9a72f668c659e38c0669352cfc0c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,197 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,199 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,201 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,203 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,203 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,204 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T12:45:24,206 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,207 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 88bc9a72f668c659e38c0669352cfc0c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62648352, jitterRate=-0.0664668083190918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T12:45:24,208 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:24,209 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 88bc9a72f668c659e38c0669352cfc0c: Running coprocessor pre-open hook at 1732538724173Writing region info on filesystem at 1732538724173Initializing all the Stores at 1732538724174 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724174Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724175 (+1 ms)Cleaning up temporary data from old regions at 1732538724203 (+28 ms)Running coprocessor post-open hooks at 1732538724208 (+5 ms)Region opened successfully at 1732538724208 2024-11-25T12:45:24,210 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., pid=17, masterSystemTime=1732538724162 2024-11-25T12:45:24,216 DEBUG [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:24,216 INFO [RS_OPEN_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:24,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=88bc9a72f668c659e38c0669352cfc0c, regionState=OPEN, openSeqNum=18, regionLocation=1ff8009e463e,44737,1732538700372 2024-11-25T12:45:24,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,44737,1732538700372 because future has completed 2024-11-25T12:45:24,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T12:45:24,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bc9a72f668c659e38c0669352cfc0c, server=1ff8009e463e,44737,1732538700372 in 215 msec 2024-11-25T12:45:24,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-25T12:45:24,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88bc9a72f668c659e38c0669352cfc0c, ASSIGN in 380 msec 2024-11-25T12:45:24,232 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 1ff8009e463e,37815,1732538700283 after splitting done 2024-11-25T12:45:24,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 1ff8009e463e,37815,1732538700283, splitWal=true, meta=false in 666 msec 2024-11-25T12:45:24,444 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c., hostname=1ff8009e463e,44737,1732538700372, seqNum=18] 2024-11-25T12:45:24,465 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 415), OpenFileDescriptor=1042 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=674 (was 646) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1489 (was 1895) 2024-11-25T12:45:24,468 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-25T12:45:24,486 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=674, ProcessCount=11, AvailableMemoryMB=1488 2024-11-25T12:45:24,487 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-25T12:45:24,512 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:24,515 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:24,517 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:24,525 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66345559, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-66345559, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:24,540 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66345559/hregion-66345559.1732538724526, exclude list is [], retry=0 2024-11-25T12:45:24,543 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:24,550 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:24,555 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:24,593 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66345559/hregion-66345559.1732538724526 2024-11-25T12:45:24,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:24,594 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 75c7cc7d9d36252e660477b7a8046159, NAME => 'testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741893_1071 (size=67) 2024-11-25T12:45:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741893_1071 (size=67) 2024-11-25T12:45:24,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741893_1071 (size=67) 2024-11-25T12:45:24,637 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:24,639 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,642 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName a 2024-11-25T12:45:24,642 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,643 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,643 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,645 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName b 2024-11-25T12:45:24,645 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,646 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,646 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,648 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName c 2024-11-25T12:45:24,648 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,649 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,652 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,653 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,653 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:24,655 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,668 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:24,669 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75c7cc7d9d36252e660477b7a8046159; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70940264, jitterRate=0.05709230899810791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:24,670 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75c7cc7d9d36252e660477b7a8046159: Writing region info on filesystem at 1732538724637Initializing all the Stores at 1732538724639 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724639Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724639Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724639Cleaning up temporary data from old regions at 1732538724653 (+14 ms)Region opened successfully at 1732538724669 (+16 ms) 2024-11-25T12:45:24,670 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75c7cc7d9d36252e660477b7a8046159, disabling compactions & flushes 2024-11-25T12:45:24,670 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:24,670 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:24,670 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. after waiting 0 ms 2024-11-25T12:45:24,670 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:24,671 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:24,671 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75c7cc7d9d36252e660477b7a8046159: Waiting for close lock at 1732538724670Disabling compacts and flushes for region at 1732538724670Disabling writes for close at 1732538724670Writing region close event to WAL at 1732538724671 (+1 ms)Closed at 1732538724671 2024-11-25T12:45:24,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741892_1070 (size=175) 2024-11-25T12:45:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741892_1070 (size=175) 2024-11-25T12:45:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741892_1070 (size=175) 2024-11-25T12:45:24,683 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:24,684 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66345559:(num 1732538724526) 2024-11-25T12:45:24,684 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:24,686 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:24,702 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, exclude list is [], retry=0 2024-11-25T12:45:24,705 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:24,705 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:24,706 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:24,758 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 2024-11-25T12:45:24,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:24,758 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75c7cc7d9d36252e660477b7a8046159, NAME => 'testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:24,759 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:24,759 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,759 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,761 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,763 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName a 2024-11-25T12:45:24,763 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,764 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,764 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,765 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName b 2024-11-25T12:45:24,766 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,766 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,766 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,767 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName c 2024-11-25T12:45:24,767 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:24,768 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:24,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,771 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,772 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,772 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,773 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:24,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:24,775 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75c7cc7d9d36252e660477b7a8046159; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59387272, jitterRate=-0.11506068706512451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:24,776 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75c7cc7d9d36252e660477b7a8046159: Writing region info on filesystem at 1732538724759Initializing all the Stores at 1732538724760 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724760Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724760Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538724760Cleaning up temporary data from old regions at 1732538724772 (+12 ms)Region opened successfully at 1732538724776 (+4 ms) 2024-11-25T12:45:24,829 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 75c7cc7d9d36252e660477b7a8046159 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T12:45:24,851 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/a/ce0eeb91a291438fac68b48a5d4328df is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732538724776/Put/seqid=0 2024-11-25T12:45:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741895_1073 (size=5958) 2024-11-25T12:45:24,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741895_1073 (size=5958) 2024-11-25T12:45:24,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741895_1073 (size=5958) 2024-11-25T12:45:24,870 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/a/ce0eeb91a291438fac68b48a5d4328df 2024-11-25T12:45:24,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/d0d2aa9c3b534b7682a86db57511ee49 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538724788/Put/seqid=0 2024-11-25T12:45:24,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741896_1074 (size=5958) 2024-11-25T12:45:24,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741896_1074 (size=5958) 2024-11-25T12:45:24,988 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/d0d2aa9c3b534b7682a86db57511ee49 2024-11-25T12:45:24,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741896_1074 (size=5958) 2024-11-25T12:45:25,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/c/7e8e9a2e9b654df59de41e03064520ea is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732538724815/Put/seqid=0 2024-11-25T12:45:25,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741897_1075 (size=5958) 2024-11-25T12:45:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741897_1075 (size=5958) 2024-11-25T12:45:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741897_1075 (size=5958) 2024-11-25T12:45:25,034 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/c/7e8e9a2e9b654df59de41e03064520ea 2024-11-25T12:45:25,041 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/a/ce0eeb91a291438fac68b48a5d4328df as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/a/ce0eeb91a291438fac68b48a5d4328df 2024-11-25T12:45:25,046 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/a/ce0eeb91a291438fac68b48a5d4328df, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:45:25,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/d0d2aa9c3b534b7682a86db57511ee49 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/d0d2aa9c3b534b7682a86db57511ee49 2024-11-25T12:45:25,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/d0d2aa9c3b534b7682a86db57511ee49, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:45:25,054 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/c/7e8e9a2e9b654df59de41e03064520ea as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/c/7e8e9a2e9b654df59de41e03064520ea 2024-11-25T12:45:25,060 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/c/7e8e9a2e9b654df59de41e03064520ea, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T12:45:25,061 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 75c7cc7d9d36252e660477b7a8046159 in 232ms, sequenceid=33, compaction requested=false 2024-11-25T12:45:25,062 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 75c7cc7d9d36252e660477b7a8046159: 2024-11-25T12:45:25,062 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75c7cc7d9d36252e660477b7a8046159, disabling compactions & flushes 2024-11-25T12:45:25,062 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:25,062 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:25,062 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. after waiting 0 ms 2024-11-25T12:45:25,062 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:25,063 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. 2024-11-25T12:45:25,063 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75c7cc7d9d36252e660477b7a8046159: Waiting for close lock at 1732538725062Disabling compacts and flushes for region at 1732538725062Disabling writes for close at 1732538725062Writing region close event to WAL at 1732538725063 (+1 ms)Closed at 1732538725063 2024-11-25T12:45:25,070 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 not finished, retry = 0 2024-11-25T12:45:25,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741894_1072 (size=6311) 2024-11-25T12:45:25,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741894_1072 (size=6311) 2024-11-25T12:45:25,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741894_1072 (size=6311) 2024-11-25T12:45:25,176 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/d0d2aa9c3b534b7682a86db57511ee49 to hdfs://localhost:41185/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/d0d2aa9c3b534b7682a86db57511ee49 2024-11-25T12:45:25,195 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, size=6.2 K (6311bytes) 2024-11-25T12:45:25,195 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 2024-11-25T12:45:25,196 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 after 1ms 2024-11-25T12:45:25,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:25,232 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 took 37ms 2024-11-25T12:45:25,242 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 so closing down 2024-11-25T12:45:25,243 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:25,247 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538724687.temp 2024-11-25T12:45:25,284 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp 2024-11-25T12:45:25,285 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:25,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741898_1076 (size=5575) 2024-11-25T12:45:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741898_1076 (size=5575) 2024-11-25T12:45:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741898_1076 (size=5575) 2024-11-25T12:45:25,299 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:25,304 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 2024-11-25T12:45:25,304 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 64 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-25T12:45:25,304 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, journal: Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, size=6.2 K (6311bytes) at 1732538725195Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 so closing down at 1732538725242 (+47 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp at 1732538725284 (+42 ms)3 split writer threads finished at 1732538725285 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538725299 (+14 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000003-wal.1732538724687.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 at 1732538725304 (+5 ms)Processed 32 edits across 1 Regions in 64 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732538725304 2024-11-25T12:45:25,307 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538724687 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538724687 2024-11-25T12:45:25,311 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 2024-11-25T12:45:25,311 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:25,319 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:25,339 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538725319, exclude list is [], retry=0 2024-11-25T12:45:25,343 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:25,344 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:25,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:25,401 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538725319 2024-11-25T12:45:25,403 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:25,404 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75c7cc7d9d36252e660477b7a8046159, NAME => 'testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:25,404 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:25,404 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,404 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,411 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,413 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName a 2024-11-25T12:45:25,414 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,421 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/a/ce0eeb91a291438fac68b48a5d4328df 2024-11-25T12:45:25,421 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,421 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,423 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName b 2024-11-25T12:45:25,423 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,424 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,424 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,425 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75c7cc7d9d36252e660477b7a8046159 columnFamilyName c 2024-11-25T12:45:25,425 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,433 DEBUG [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/c/7e8e9a2e9b654df59de41e03064520ea 2024-11-25T12:45:25,433 INFO [StoreOpener-75c7cc7d9d36252e660477b7a8046159-1 {}] regionserver.HStore(327): Store=75c7cc7d9d36252e660477b7a8046159/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,433 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,439 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,440 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 2024-11-25T12:45:25,479 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:25,481 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 2024-11-25T12:45:25,482 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 75c7cc7d9d36252e660477b7a8046159 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T12:45:25,503 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/bc9cf026a2dc4297b814853011f7421b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538724788/Put/seqid=0 2024-11-25T12:45:25,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:45:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:45:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741900_1078 (size=5958) 2024-11-25T12:45:25,522 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/bc9cf026a2dc4297b814853011f7421b 2024-11-25T12:45:25,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/.tmp/b/bc9cf026a2dc4297b814853011f7421b as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/bc9cf026a2dc4297b814853011f7421b 2024-11-25T12:45:25,555 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/b/bc9cf026a2dc4297b814853011f7421b, entries=10, sequenceid=32, filesize=5.8 K 2024-11-25T12:45:25,555 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 75c7cc7d9d36252e660477b7a8046159 in 73ms, sequenceid=32, compaction requested=false; wal=null 2024-11-25T12:45:25,557 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/0000000000000000032 2024-11-25T12:45:25,562 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,562 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,563 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:25,566 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75c7cc7d9d36252e660477b7a8046159 2024-11-25T12:45:25,569 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/75c7cc7d9d36252e660477b7a8046159/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-25T12:45:25,571 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75c7cc7d9d36252e660477b7a8046159; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65192939, jitterRate=-0.02854950726032257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:25,573 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75c7cc7d9d36252e660477b7a8046159: Writing region info on filesystem at 1732538725404Initializing all the Stores at 1732538725406 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725406Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725411 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725411Obtaining lock to block concurrent updates at 1732538725482 (+71 ms)Preparing flush snapshotting stores in 75c7cc7d9d36252e660477b7a8046159 at 1732538725482Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732538725482Flushing stores of testReplayEditsWrittenViaHRegion,,1732538724513.75c7cc7d9d36252e660477b7a8046159. at 1732538725482Flushing 75c7cc7d9d36252e660477b7a8046159/b: creating writer at 1732538725482Flushing 75c7cc7d9d36252e660477b7a8046159/b: appending metadata at 1732538725502 (+20 ms)Flushing 75c7cc7d9d36252e660477b7a8046159/b: closing flushed file at 1732538725502Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23dd2fbf: reopening flushed file at 1732538725540 (+38 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 75c7cc7d9d36252e660477b7a8046159 in 73ms, sequenceid=32, compaction requested=false; wal=null at 1732538725555 (+15 ms)Cleaning up temporary data from old regions at 1732538725562 (+7 ms)Region opened successfully at 1732538725572 (+10 ms) 2024-11-25T12:45:25,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=420 (was 411) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:40858 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:45926 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:48668 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:35770 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:53992 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:32870 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=674 (was 674), ProcessCount=11 (was 11), AvailableMemoryMB=1467 (was 1488) 2024-11-25T12:45:25,625 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-25T12:45:25,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=674, ProcessCount=11, AvailableMemoryMB=1462 2024-11-25T12:45:25,650 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-25T12:45:25,668 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:25,671 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:25,673 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:25,676 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25108020, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-25108020, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:25,689 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25108020/hregion-25108020.1732538725676, exclude list is [], retry=0 2024-11-25T12:45:25,692 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:25,694 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:25,699 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:25,733 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25108020/hregion-25108020.1732538725676 2024-11-25T12:45:25,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:25,734 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 81be2ef6123f61caf1bdd345602e59e8, NAME => 'testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741902_1080 (size=68) 2024-11-25T12:45:25,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741902_1080 (size=68) 2024-11-25T12:45:25,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741902_1080 (size=68) 2024-11-25T12:45:25,757 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:25,759 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,761 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName a 2024-11-25T12:45:25,761 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,762 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,762 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,764 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName b 2024-11-25T12:45:25,764 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,764 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,765 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,766 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName c 2024-11-25T12:45:25,766 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,767 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,767 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,770 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,770 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,770 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:25,772 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,774 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:25,774 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81be2ef6123f61caf1bdd345602e59e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74901072, jitterRate=0.11611294746398926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81be2ef6123f61caf1bdd345602e59e8: Writing region info on filesystem at 1732538725757Initializing all the Stores at 1732538725758 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725758Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725759 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725759Cleaning up temporary data from old regions at 1732538725770 (+11 ms)Region opened successfully at 1732538725775 (+5 ms) 2024-11-25T12:45:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81be2ef6123f61caf1bdd345602e59e8, disabling compactions & flushes 2024-11-25T12:45:25,775 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. after waiting 0 ms 2024-11-25T12:45:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:25,776 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:25,776 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81be2ef6123f61caf1bdd345602e59e8: Waiting for close lock at 1732538725775Disabling compacts and flushes for region at 1732538725775Disabling writes for close at 1732538725775Writing region close event to WAL at 1732538725776 (+1 ms)Closed at 1732538725776 2024-11-25T12:45:25,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741901_1079 (size=175) 2024-11-25T12:45:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741901_1079 (size=175) 2024-11-25T12:45:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741901_1079 (size=175) 2024-11-25T12:45:25,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:25,781 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25108020:(num 1732538725676) 2024-11-25T12:45:25,781 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:25,783 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:25,795 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, exclude list is [], retry=0 2024-11-25T12:45:25,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:25,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:25,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:25,824 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 2024-11-25T12:45:25,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:25,889 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 81be2ef6123f61caf1bdd345602e59e8, NAME => 'testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:25,892 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,892 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:25,892 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,892 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,895 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,896 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName a 2024-11-25T12:45:25,896 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,897 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,897 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,899 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName b 2024-11-25T12:45:25,899 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,900 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,900 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,902 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName c 2024-11-25T12:45:25,902 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:25,903 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:25,903 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,904 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,908 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,908 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,908 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:25,910 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,911 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81be2ef6123f61caf1bdd345602e59e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61417207, jitterRate=-0.08481229841709137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:25,912 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:25,912 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81be2ef6123f61caf1bdd345602e59e8: Running coprocessor pre-open hook at 1732538725892Writing region info on filesystem at 1732538725892Initializing all the Stores at 1732538725894 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725894Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725894Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538725894Cleaning up temporary data from old regions at 1732538725908 (+14 ms)Running coprocessor post-open hooks at 1732538725912 (+4 ms)Region opened successfully at 1732538725912 2024-11-25T12:45:25,927 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81be2ef6123f61caf1bdd345602e59e8 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-25T12:45:25,933 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:26,256 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:45:26,934 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:27,934 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:28,031 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:45:28,935 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:29,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-25T12:45:29,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:29,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T12:45:29,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:29,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-25T12:45:29,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:29,936 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:30,937 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:31,394 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:45:31,937 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:32,938 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:33,939 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:34,940 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 81be2ef6123f61caf1bdd345602e59e8/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:34,942 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 81be2ef6123f61caf1bdd345602e59e8: 2024-11-25T12:45:34,943 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,963 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 81be2ef6123f61caf1bdd345602e59e8: 2024-11-25T12:45:34,963 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-25T12:45:34,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81be2ef6123f61caf1bdd345602e59e8, disabling compactions & flushes 2024-11-25T12:45:34,963 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,963 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,963 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. after waiting 0 ms 2024-11-25T12:45:34,963 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,964 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,964 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. 2024-11-25T12:45:34,964 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81be2ef6123f61caf1bdd345602e59e8: Waiting for close lock at 1732538734963Running coprocessor pre-close hooks at 1732538734963Disabling compacts and flushes for region at 1732538734963Disabling writes for close at 1732538734963Writing region close event to WAL at 1732538734964 (+1 ms)Running coprocessor post-close hooks at 1732538734964Closed at 1732538734964 2024-11-25T12:45:34,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741903_1081 (size=4236) 2024-11-25T12:45:34,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741903_1081 (size=4236) 2024-11-25T12:45:34,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741903_1081 (size=4236) 2024-11-25T12:45:34,991 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, size=4.1 K (4236bytes) 2024-11-25T12:45:34,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 2024-11-25T12:45:34,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 after 0ms 2024-11-25T12:45:35,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:35,017 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 took 26ms 2024-11-25T12:45:35,019 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 so closing down 2024-11-25T12:45:35,019 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:35,021 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732538725783.temp 2024-11-25T12:45:35,045 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp 2024-11-25T12:45:35,045 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741904_1082 (size=3245) 2024-11-25T12:45:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741904_1082 (size=3245) 2024-11-25T12:45:35,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741904_1082 (size=3245) 2024-11-25T12:45:35,053 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:35,055 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 2024-11-25T12:45:35,055 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-25T12:45:35,055 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, journal: Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, size=4.1 K (4236bytes) at 1732538734991Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 so closing down at 1732538735019 (+28 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp at 1732538735045 (+26 ms)3 split writer threads finished at 1732538735045Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732538735053 (+8 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000004-wal.1732538725783.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 at 1732538735055 (+2 ms)Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732538735055 2024-11-25T12:45:35,057 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538725783 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538725783 2024-11-25T12:45:35,058 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 2024-11-25T12:45:35,058 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:35,060 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:35,078 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538735061, exclude list is [], retry=0 2024-11-25T12:45:35,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:35,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:35,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:35,105 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538735061 2024-11-25T12:45:35,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:35,106 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 81be2ef6123f61caf1bdd345602e59e8, NAME => 'testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:35,106 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,106 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:35,106 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,106 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,108 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,109 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName a 2024-11-25T12:45:35,109 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:35,110 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:35,110 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,110 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName b 2024-11-25T12:45:35,110 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:35,111 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:35,111 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,112 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81be2ef6123f61caf1bdd345602e59e8 columnFamilyName c 2024-11-25T12:45:35,112 DEBUG [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:35,112 INFO [StoreOpener-81be2ef6123f61caf1bdd345602e59e8-1 {}] regionserver.HStore(327): Store=81be2ef6123f61caf1bdd345602e59e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:35,112 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,114 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,114 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 2024-11-25T12:45:35,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:35,141 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 2024-11-25T12:45:35,142 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81be2ef6123f61caf1bdd345602e59e8 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-25T12:45:35,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/a/5e9ea1ff80404c8fbbe22a0cb0babad5 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732538734951/Put/seqid=0 2024-11-25T12:45:35,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741906_1084 (size=5523) 2024-11-25T12:45:35,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741906_1084 (size=5523) 2024-11-25T12:45:35,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741906_1084 (size=5523) 2024-11-25T12:45:35,165 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/a/5e9ea1ff80404c8fbbe22a0cb0babad5 2024-11-25T12:45:35,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/b/82c0d0cc7fcb4c14b82668b4055d2f45 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732538734943/Put/seqid=0 2024-11-25T12:45:35,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741907_1085 (size=5524) 2024-11-25T12:45:35,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741907_1085 (size=5524) 2024-11-25T12:45:35,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741907_1085 (size=5524) 2024-11-25T12:45:35,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/b/82c0d0cc7fcb4c14b82668b4055d2f45 2024-11-25T12:45:35,219 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/c/5fd01e5179b540a7a1d73e5ebee32ff5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732538734948/Put/seqid=0 2024-11-25T12:45:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741908_1086 (size=5457) 2024-11-25T12:45:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741908_1086 (size=5457) 2024-11-25T12:45:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741908_1086 (size=5457) 2024-11-25T12:45:35,226 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/c/5fd01e5179b540a7a1d73e5ebee32ff5 2024-11-25T12:45:35,245 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/a/5e9ea1ff80404c8fbbe22a0cb0babad5 as hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/a/5e9ea1ff80404c8fbbe22a0cb0babad5 2024-11-25T12:45:35,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/a/5e9ea1ff80404c8fbbe22a0cb0babad5, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T12:45:35,253 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/b/82c0d0cc7fcb4c14b82668b4055d2f45 as hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/b/82c0d0cc7fcb4c14b82668b4055d2f45 2024-11-25T12:45:35,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/b/82c0d0cc7fcb4c14b82668b4055d2f45, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T12:45:35,273 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/.tmp/c/5fd01e5179b540a7a1d73e5ebee32ff5 as hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/c/5fd01e5179b540a7a1d73e5ebee32ff5 2024-11-25T12:45:35,288 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/c/5fd01e5179b540a7a1d73e5ebee32ff5, entries=6, sequenceid=26, filesize=5.3 K 2024-11-25T12:45:35,288 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 81be2ef6123f61caf1bdd345602e59e8 in 146ms, sequenceid=26, compaction requested=false; wal=null 2024-11-25T12:45:35,289 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/0000000000000000026 2024-11-25T12:45:35,290 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,290 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,291 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:35,293 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,295 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsAfterAbortingFlush/81be2ef6123f61caf1bdd345602e59e8/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-25T12:45:35,296 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81be2ef6123f61caf1bdd345602e59e8; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62470063, jitterRate=-0.0691235214471817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:35,296 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81be2ef6123f61caf1bdd345602e59e8 2024-11-25T12:45:35,297 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81be2ef6123f61caf1bdd345602e59e8: Running coprocessor pre-open hook at 1732538735107Writing region info on filesystem at 1732538735107Initializing all the Stores at 1732538735108 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538735108Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538735108Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538735108Obtaining lock to block concurrent updates at 1732538735142 (+34 ms)Preparing flush snapshotting stores in 81be2ef6123f61caf1bdd345602e59e8 at 1732538735142Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732538735142Flushing stores of testReplayEditsAfterAbortingFlush,,1732538725670.81be2ef6123f61caf1bdd345602e59e8. at 1732538735142Flushing 81be2ef6123f61caf1bdd345602e59e8/a: creating writer at 1732538735142Flushing 81be2ef6123f61caf1bdd345602e59e8/a: appending metadata at 1732538735158 (+16 ms)Flushing 81be2ef6123f61caf1bdd345602e59e8/a: closing flushed file at 1732538735158Flushing 81be2ef6123f61caf1bdd345602e59e8/b: creating writer at 1732538735170 (+12 ms)Flushing 81be2ef6123f61caf1bdd345602e59e8/b: appending metadata at 1732538735188 (+18 ms)Flushing 81be2ef6123f61caf1bdd345602e59e8/b: closing flushed file at 1732538735188Flushing 81be2ef6123f61caf1bdd345602e59e8/c: creating writer at 1732538735202 (+14 ms)Flushing 81be2ef6123f61caf1bdd345602e59e8/c: appending metadata at 1732538735219 (+17 ms)Flushing 81be2ef6123f61caf1bdd345602e59e8/c: closing flushed file at 1732538735219Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41bb3c29: reopening flushed file at 1732538735243 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31096a2: reopening flushed file at 1732538735252 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358880: reopening flushed file at 1732538735268 (+16 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 81be2ef6123f61caf1bdd345602e59e8 in 146ms, sequenceid=26, compaction requested=false; wal=null at 1732538735288 (+20 ms)Cleaning up temporary data from old regions at 1732538735290 (+2 ms)Running coprocessor post-open hooks at 1732538735296 (+6 ms)Region opened successfully at 1732538735297 (+1 ms) 2024-11-25T12:45:35,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=423 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:43946 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58060 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49650 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58082 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:43930 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49696 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=616 (was 674), ProcessCount=11 (was 11), AvailableMemoryMB=1335 (was 1462) 2024-11-25T12:45:35,319 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-25T12:45:35,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=423, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=616, ProcessCount=11, AvailableMemoryMB=1334 2024-11-25T12:45:35,331 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-25T12:45:35,348 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:35,349 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:35,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:35,352 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93587889, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-93587889, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:35,363 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93587889/hregion-93587889.1732538735352, exclude list is [], retry=0 2024-11-25T12:45:35,366 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:35,366 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:35,367 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:35,415 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93587889/hregion-93587889.1732538735352 2024-11-25T12:45:35,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:35,415 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bb99ebff4f7a1f8e07e9672f7b9302cd, NAME => 'testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741910_1088 (size=61) 2024-11-25T12:45:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741910_1088 (size=61) 2024-11-25T12:45:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741910_1088 (size=61) 2024-11-25T12:45:35,425 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:35,426 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,427 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb99ebff4f7a1f8e07e9672f7b9302cd columnFamilyName a 2024-11-25T12:45:35,427 DEBUG [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:35,427 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(327): Store=bb99ebff4f7a1f8e07e9672f7b9302cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:35,428 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,429 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,429 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,430 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,432 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:35,433 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb99ebff4f7a1f8e07e9672f7b9302cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58898027, jitterRate=-0.12235100567340851}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Writing region info on filesystem at 1732538735425Initializing all the Stores at 1732538735425Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538735425Cleaning up temporary data from old regions at 1732538735429 (+4 ms)Region opened successfully at 1732538735434 (+5 ms) 2024-11-25T12:45:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bb99ebff4f7a1f8e07e9672f7b9302cd, disabling compactions & flushes 2024-11-25T12:45:35,434 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. after waiting 0 ms 2024-11-25T12:45:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,435 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,435 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Waiting for close lock at 1732538735434Disabling compacts and flushes for region at 1732538735434Disabling writes for close at 1732538735434Writing region close event to WAL at 1732538735434Closed at 1732538735435 (+1 ms) 2024-11-25T12:45:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741909_1087 (size=175) 2024-11-25T12:45:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741909_1087 (size=175) 2024-11-25T12:45:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741909_1087 (size=175) 2024-11-25T12:45:35,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:35,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93587889:(num 1732538735352) 2024-11-25T12:45:35,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:35,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:35,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, exclude list is [], retry=0 2024-11-25T12:45:35,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:35,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:35,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:35,477 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 2024-11-25T12:45:35,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:35,478 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bb99ebff4f7a1f8e07e9672f7b9302cd, NAME => 'testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:35,478 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:35,478 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,478 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,480 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,480 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb99ebff4f7a1f8e07e9672f7b9302cd columnFamilyName a 2024-11-25T12:45:35,480 DEBUG [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:35,481 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(327): Store=bb99ebff4f7a1f8e07e9672f7b9302cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:35,481 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,483 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,483 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,485 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:35,485 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb99ebff4f7a1f8e07e9672f7b9302cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70372434, jitterRate=0.048630982637405396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:35,486 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Writing region info on filesystem at 1732538735478Initializing all the Stores at 1732538735479 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538735479Cleaning up temporary data from old regions at 1732538735483 (+4 ms)Region opened successfully at 1732538735486 (+3 ms) 2024-11-25T12:45:35,494 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bb99ebff4f7a1f8e07e9672f7b9302cd, disabling compactions & flushes 2024-11-25T12:45:35,494 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,494 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,494 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. after waiting 0 ms 2024-11-25T12:45:35,494 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,494 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,494 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. 2024-11-25T12:45:35,494 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Waiting for close lock at 1732538735494Disabling compacts and flushes for region at 1732538735494Disabling writes for close at 1732538735494Writing region close event to WAL at 1732538735494Closed at 1732538735494 2024-11-25T12:45:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741911_1089 (size=1795) 2024-11-25T12:45:35,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741911_1089 (size=1795) 2024-11-25T12:45:35,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741911_1089 (size=1795) 2024-11-25T12:45:35,511 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, size=1.8 K (1795bytes) 2024-11-25T12:45:35,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 2024-11-25T12:45:35,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 after 0ms 2024-11-25T12:45:35,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:35,535 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 took 24ms 2024-11-25T12:45:35,537 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 so closing down 2024-11-25T12:45:35,537 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:35,538 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538735441.temp 2024-11-25T12:45:35,561 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp 2024-11-25T12:45:35,562 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:35,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741912_1090 (size=1795) 2024-11-25T12:45:35,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741912_1090 (size=1795) 2024-11-25T12:45:35,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741912_1090 (size=1795) 2024-11-25T12:45:35,568 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:35,570 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp to hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 2024-11-25T12:45:35,570 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-25T12:45:35,570 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, journal: Splitting hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, size=1.8 K (1795bytes) at 1732538735511Finishing writing output for hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 so closing down at 1732538735537 (+26 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp at 1732538735561 (+24 ms)3 split writer threads finished at 1732538735562 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732538735569 (+7 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000003-wal.1732538735441.temp to hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 at 1732538735570 (+1 ms)Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732538735570 2024-11-25T12:45:35,572 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735441 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538735441 2024-11-25T12:45:35,573 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 2024-11-25T12:45:35,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:35,976 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:35,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:35,990 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735978, exclude list is [], retry=0 2024-11-25T12:45:35,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:35,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:35,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:36,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735978 2024-11-25T12:45:36,017 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:36,017 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bb99ebff4f7a1f8e07e9672f7b9302cd, NAME => 'testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:36,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:36,017 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,017 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,020 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,022 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb99ebff4f7a1f8e07e9672f7b9302cd columnFamilyName a 2024-11-25T12:45:36,022 DEBUG [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,023 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(327): Store=bb99ebff4f7a1f8e07e9672f7b9302cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,023 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,026 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 2024-11-25T12:45:36,052 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:36,053 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 2024-11-25T12:45:36,053 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb99ebff4f7a1f8e07e9672f7b9302cd 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-25T12:45:36,069 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/.tmp/a/76c2319965154b3e95230cfe8795f822 is 79, key is testDatalossWhenInputError/a:x0/1732538735486/Put/seqid=0 2024-11-25T12:45:36,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741914_1092 (size=5808) 2024-11-25T12:45:36,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741914_1092 (size=5808) 2024-11-25T12:45:36,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741914_1092 (size=5808) 2024-11-25T12:45:36,075 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/.tmp/a/76c2319965154b3e95230cfe8795f822 2024-11-25T12:45:36,084 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/.tmp/a/76c2319965154b3e95230cfe8795f822 as hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/a/76c2319965154b3e95230cfe8795f822 2024-11-25T12:45:36,092 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/a/76c2319965154b3e95230cfe8795f822, entries=10, sequenceid=12, filesize=5.7 K 2024-11-25T12:45:36,092 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for bb99ebff4f7a1f8e07e9672f7b9302cd in 39ms, sequenceid=12, compaction requested=false; wal=null 2024-11-25T12:45:36,093 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/0000000000000000012 2024-11-25T12:45:36,094 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,094 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,096 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,098 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-25T12:45:36,099 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb99ebff4f7a1f8e07e9672f7b9302cd; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66674060, jitterRate=-0.006479084491729736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:36,100 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Writing region info on filesystem at 1732538736017Initializing all the Stores at 1732538736020 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736020Obtaining lock to block concurrent updates at 1732538736053 (+33 ms)Preparing flush snapshotting stores in bb99ebff4f7a1f8e07e9672f7b9302cd at 1732538736053Finished memstore snapshotting testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732538736053Flushing stores of testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd. at 1732538736053Flushing bb99ebff4f7a1f8e07e9672f7b9302cd/a: creating writer at 1732538736053Flushing bb99ebff4f7a1f8e07e9672f7b9302cd/a: appending metadata at 1732538736068 (+15 ms)Flushing bb99ebff4f7a1f8e07e9672f7b9302cd/a: closing flushed file at 1732538736068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f4b5576: reopening flushed file at 1732538736082 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for bb99ebff4f7a1f8e07e9672f7b9302cd in 39ms, sequenceid=12, compaction requested=false; wal=null at 1732538736092 (+10 ms)Cleaning up temporary data from old regions at 1732538736094 (+2 ms)Region opened successfully at 1732538736100 (+6 ms) 2024-11-25T12:45:36,103 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bb99ebff4f7a1f8e07e9672f7b9302cd, NAME => 'testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:36,103 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732538735348.bb99ebff4f7a1f8e07e9672f7b9302cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:36,103 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,103 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,104 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,105 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb99ebff4f7a1f8e07e9672f7b9302cd columnFamilyName a 2024-11-25T12:45:36,105 DEBUG [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,111 DEBUG [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/a/76c2319965154b3e95230cfe8795f822 2024-11-25T12:45:36,111 INFO [StoreOpener-bb99ebff4f7a1f8e07e9672f7b9302cd-1 {}] regionserver.HStore(327): Store=bb99ebff4f7a1f8e07e9672f7b9302cd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,111 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,112 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,114 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,114 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,116 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb99ebff4f7a1f8e07e9672f7b9302cd 2024-11-25T12:45:36,118 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testDatalossWhenInputError/bb99ebff4f7a1f8e07e9672f7b9302cd/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-25T12:45:36,119 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb99ebff4f7a1f8e07e9672f7b9302cd; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72039159, jitterRate=0.07346712052822113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T12:45:36,119 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb99ebff4f7a1f8e07e9672f7b9302cd: Writing region info on filesystem at 1732538736103Initializing all the Stores at 1732538736104 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736104Cleaning up temporary data from old regions at 1732538736114 (+10 ms)Region opened successfully at 1732538736119 (+5 ms) 2024-11-25T12:45:36,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=432 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:43946 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58060 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58158 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:44020 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49728 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49696 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-322817772-172.17.0.3-1732538693888:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=616 (was 616), ProcessCount=11 (was 11), AvailableMemoryMB=1315 (was 1334) 2024-11-25T12:45:36,138 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-25T12:45:36,152 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=432, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=616, ProcessCount=11, AvailableMemoryMB=1314 2024-11-25T12:45:36,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-25T12:45:36,168 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:36,171 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:36,172 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:36,174 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47002500, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-47002500, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:36,186 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47002500/hregion-47002500.1732538736174, exclude list is [], retry=0 2024-11-25T12:45:36,188 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:36,188 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:36,188 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:36,217 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47002500/hregion-47002500.1732538736174 2024-11-25T12:45:36,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:36,217 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c32549cd7f31e798e52fad0b55b0d992, NAME => 'testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:36,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741916_1094 (size=63) 2024-11-25T12:45:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741916_1094 (size=63) 2024-11-25T12:45:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741916_1094 (size=63) 2024-11-25T12:45:36,227 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:36,229 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,230 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName a 2024-11-25T12:45:36,230 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,231 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,231 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,232 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName b 2024-11-25T12:45:36,232 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,233 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,233 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,234 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName c 2024-11-25T12:45:36,234 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,234 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,235 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,237 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,237 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,237 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:36,239 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,241 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:36,241 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c32549cd7f31e798e52fad0b55b0d992; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64411970, jitterRate=-0.04018685221672058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c32549cd7f31e798e52fad0b55b0d992: Writing region info on filesystem at 1732538736228Initializing all the Stores at 1732538736228Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736229 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736229Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736229Cleaning up temporary data from old regions at 1732538736237 (+8 ms)Region opened successfully at 1732538736242 (+5 ms) 2024-11-25T12:45:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c32549cd7f31e798e52fad0b55b0d992, disabling compactions & flushes 2024-11-25T12:45:36,242 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. after waiting 0 ms 2024-11-25T12:45:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:36,243 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:36,243 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c32549cd7f31e798e52fad0b55b0d992: Waiting for close lock at 1732538736242Disabling compacts and flushes for region at 1732538736242Disabling writes for close at 1732538736242Writing region close event to WAL at 1732538736243 (+1 ms)Closed at 1732538736243 2024-11-25T12:45:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741915_1093 (size=175) 2024-11-25T12:45:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741915_1093 (size=175) 2024-11-25T12:45:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741915_1093 (size=175) 2024-11-25T12:45:36,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:36,252 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47002500:(num 1732538736174) 2024-11-25T12:45:36,252 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:36,254 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:36,269 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, exclude list is [], retry=0 2024-11-25T12:45:36,271 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:36,271 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:36,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:36,296 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 2024-11-25T12:45:36,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865)] 2024-11-25T12:45:36,297 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c32549cd7f31e798e52fad0b55b0d992, NAME => 'testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:36,297 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:36,297 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,297 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,298 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,299 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName a 2024-11-25T12:45:36,299 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,300 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,300 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,300 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName b 2024-11-25T12:45:36,301 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,301 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,301 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,302 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName c 2024-11-25T12:45:36,302 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:36,302 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:36,302 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,303 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,305 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,305 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,306 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:36,307 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:36,307 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c32549cd7f31e798e52fad0b55b0d992; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66880142, jitterRate=-0.0034082233905792236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:36,308 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c32549cd7f31e798e52fad0b55b0d992: Writing region info on filesystem at 1732538736297Initializing all the Stores at 1732538736298 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736298Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736298Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538736298Cleaning up temporary data from old regions at 1732538736305 (+7 ms)Region opened successfully at 1732538736308 (+3 ms) 2024-11-25T12:45:36,312 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732538736311/Put/seqid=0 2024-11-25T12:45:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741918_1096 (size=4875) 2024-11-25T12:45:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741918_1096 (size=4875) 2024-11-25T12:45:36,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741918_1096 (size=4875) 2024-11-25T12:45:36,323 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732538736322/Put/seqid=0 2024-11-25T12:45:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741919_1097 (size=4875) 2024-11-25T12:45:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741919_1097 (size=4875) 2024-11-25T12:45:36,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741919_1097 (size=4875) 2024-11-25T12:45:36,336 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732538736336/Put/seqid=0 2024-11-25T12:45:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741920_1098 (size=4875) 2024-11-25T12:45:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741920_1098 (size=4875) 2024-11-25T12:45:36,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741920_1098 (size=4875) 2024-11-25T12:45:36,349 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,354 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-25T12:45:36,354 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:45:36,354 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,358 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-25T12:45:36,358 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:45:36,359 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,364 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-25T12:45:36,364 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T12:45:36,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c32549cd7f31e798e52fad0b55b0d992 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-25T12:45:36,382 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp/a/8b29217c0d444488be65203ac8acd52f is 55, key is testCompactedBulkLoadedFiles/a:a/1732538736308/Put/seqid=0 2024-11-25T12:45:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741921_1099 (size=5107) 2024-11-25T12:45:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741921_1099 (size=5107) 2024-11-25T12:45:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741921_1099 (size=5107) 2024-11-25T12:45:36,394 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp/a/8b29217c0d444488be65203ac8acd52f 2024-11-25T12:45:36,401 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp/a/8b29217c0d444488be65203ac8acd52f as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f 2024-11-25T12:45:36,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f, entries=1, sequenceid=4, filesize=5.0 K 2024-11-25T12:45:36,408 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c32549cd7f31e798e52fad0b55b0d992 in 44ms, sequenceid=4, compaction requested=false 2024-11-25T12:45:36,408 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c32549cd7f31e798e52fad0b55b0d992: 2024-11-25T12:45:36,409 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ 2024-11-25T12:45:36,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ 2024-11-25T12:45:36,412 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ 2024-11-25T12:45:36,412 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile0 into c32549cd7f31e798e52fad0b55b0d992/a as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ - updating store file list. 2024-11-25T12:45:36,418 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:36,418 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ into c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,418 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile0 into c32549cd7f31e798e52fad0b55b0d992/a (new location: hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_) 2024-11-25T12:45:36,420 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile1 into c32549cd7f31e798e52fad0b55b0d992/a as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ - updating store file list. 2024-11-25T12:45:36,425 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f2a72f849318415da0bbdda57da1679c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:36,425 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ into c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,425 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile1 into c32549cd7f31e798e52fad0b55b0d992/a (new location: hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_) 2024-11-25T12:45:36,426 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile2 into c32549cd7f31e798e52fad0b55b0d992/a as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ - updating store file list. 2024-11-25T12:45:36,431 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 30191c0bdc7a4957ad73aa67419066fe_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:36,431 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ into c32549cd7f31e798e52fad0b55b0d992/a 2024-11-25T12:45:36,431 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41185/hbase/testCompactedBulkLoadedFiles/hfile2 into c32549cd7f31e798e52fad0b55b0d992/a (new location: hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_) 2024-11-25T12:45:36,440 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:45:36,440 DEBUG [Time-limited test {}] regionserver.HStore(1541): c32549cd7f31e798e52fad0b55b0d992/a is initiating major compaction (all files) 2024-11-25T12:45:36,440 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c32549cd7f31e798e52fad0b55b0d992/a in testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:36,440 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_] into tmpdir=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp, totalSize=19.3 K 2024-11-25T12:45:36,441 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8b29217c0d444488be65203ac8acd52f, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732538736308 2024-11-25T12:45:36,441 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:45:36,442 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f2a72f849318415da0bbdda57da1679c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:45:36,442 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 30191c0bdc7a4957ad73aa67419066fe_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T12:45:36,456 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp/a/8fd59d07baac494aac53791a8de068ea is 55, key is testCompactedBulkLoadedFiles/a:a/1732538736308/Put/seqid=0 2024-11-25T12:45:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741922_1100 (size=6154) 2024-11-25T12:45:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741922_1100 (size=6154) 2024-11-25T12:45:36,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741922_1100 (size=6154) 2024-11-25T12:45:36,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/.tmp/a/8fd59d07baac494aac53791a8de068ea as hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8fd59d07baac494aac53791a8de068ea 2024-11-25T12:45:36,476 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c32549cd7f31e798e52fad0b55b0d992/a of c32549cd7f31e798e52fad0b55b0d992 into 8fd59d07baac494aac53791a8de068ea(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:45:36,476 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c32549cd7f31e798e52fad0b55b0d992: 2024-11-25T12:45:36,476 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T12:45:36,476 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T12:45:36,512 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, size=0 (0bytes) 2024-11-25T12:45:36,512 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 might be still open, length is 0 2024-11-25T12:45:36,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 2024-11-25T12:45:36,512 WARN [IPC Server handler 3 on default port 41185 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-25T12:45:36,513 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 after 1ms 2024-11-25T12:45:36,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58192 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35695:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58192 dst: /127.0.0.1:35695 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35695 remote=/127.0.0.1:58192]. Total timeout mills is 60000, 59493 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:36,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:44060 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44060 dst: /127.0.0.1:38775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:36,983 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49768 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45631:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49768 dst: /127.0.0.1:45631 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:36,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741917_1101 (size=1794) 2024-11-25T12:45:36,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741917_1101 (size=1794) 2024-11-25T12:45:39,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-25T12:45:39,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:39,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-25T12:45:39,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-25T12:45:40,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:45:40,513 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 after 4001ms 2024-11-25T12:45:40,538 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:40,538 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 took 4027ms 2024-11-25T12:45:40,540 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255; continuing. 2024-11-25T12:45:40,541 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 so closing down 2024-11-25T12:45:40,541 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:40,542 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538736255.temp 2024-11-25T12:45:40,567 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp 2024-11-25T12:45:40,567 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741923_1102 (size=800) 2024-11-25T12:45:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741923_1102 (size=800) 2024-11-25T12:45:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741923_1102 (size=800) 2024-11-25T12:45:40,574 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:40,575 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp to hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 2024-11-25T12:45:40,575 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:45:40,575 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, journal: Splitting hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, size=0 (0bytes) at 1732538736512Finishing writing output for hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 so closing down at 1732538740541 (+4029 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp at 1732538740567 (+26 ms)3 split writer threads finished at 1732538740567Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732538740574 (+7 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000003-wal.1732538736255.temp to hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 at 1732538740575 (+1 ms)Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255, size=0, length=0, corrupted=false, cancelled=false at 1732538740575 2024-11-25T12:45:40,577 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538736255 2024-11-25T12:45:40,577 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 2024-11-25T12:45:40,577 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:40,579 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:40,591 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538740579, exclude list is [], retry=0 2024-11-25T12:45:40,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:40,594 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:40,594 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:40,617 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538740579 2024-11-25T12:45:40,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:40,617 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c32549cd7f31e798e52fad0b55b0d992, NAME => 'testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:40,617 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:40,617 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,617 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,619 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,620 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName a 2024-11-25T12:45:40,620 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,625 DEBUG [StoreFileOpener-c32549cd7f31e798e52fad0b55b0d992-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 30191c0bdc7a4957ad73aa67419066fe_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:40,626 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ 2024-11-25T12:45:40,629 DEBUG [StoreFileOpener-c32549cd7f31e798e52fad0b55b0d992-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:40,630 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ 2024-11-25T12:45:40,634 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f 2024-11-25T12:45:40,638 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8fd59d07baac494aac53791a8de068ea 2024-11-25T12:45:40,642 DEBUG [StoreFileOpener-c32549cd7f31e798e52fad0b55b0d992-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f2a72f849318415da0bbdda57da1679c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T12:45:40,642 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ 2024-11-25T12:45:40,642 WARN [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@297698bf 2024-11-25T12:45:40,642 WARN [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@297698bf 2024-11-25T12:45:40,642 WARN [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@297698bf 2024-11-25T12:45:40,642 WARN [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@297698bf 2024-11-25T12:45:40,642 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_] to archive 2024-11-25T12:45:40,643 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:45:40,645 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ to hdfs://localhost:41185/hbase/archive/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/30191c0bdc7a4957ad73aa67419066fe_SeqId_4_ 2024-11-25T12:45:40,646 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ to hdfs://localhost:41185/hbase/archive/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_ 2024-11-25T12:45:40,648 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f to hdfs://localhost:41185/hbase/archive/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/8b29217c0d444488be65203ac8acd52f 2024-11-25T12:45:40,649 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ to hdfs://localhost:41185/hbase/archive/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/a/f2a72f849318415da0bbdda57da1679c_SeqId_4_ 2024-11-25T12:45:40,649 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,649 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,650 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName b 2024-11-25T12:45:40,650 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,650 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,650 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,651 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32549cd7f31e798e52fad0b55b0d992 columnFamilyName c 2024-11-25T12:45:40,651 DEBUG [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,651 INFO [StoreOpener-c32549cd7f31e798e52fad0b55b0d992-1 {}] regionserver.HStore(327): Store=c32549cd7f31e798e52fad0b55b0d992/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,654 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 2024-11-25T12:45:40,678 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:40,680 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c32549cd7f31e798e52fad0b55b0d992 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c32549cd7f31e798e52fad0b55b0d992" family_name: "a" compaction_input: "8b29217c0d444488be65203ac8acd52f" compaction_input: "74e62fce0e4047dcbc8f4e5add87adde_SeqId_4_" compaction_input: "f2a72f849318415da0bbdda57da1679c_SeqId_4_" compaction_input: "30191c0bdc7a4957ad73aa67419066fe_SeqId_4_" compaction_output: "8fd59d07baac494aac53791a8de068ea" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-25T12:45:40,680 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-25T12:45:40,680 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 2024-11-25T12:45:40,681 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/0000000000000000008 2024-11-25T12:45:40,682 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,682 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,682 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:40,683 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c32549cd7f31e798e52fad0b55b0d992 2024-11-25T12:45:40,685 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testCompactedBulkLoadedFiles/c32549cd7f31e798e52fad0b55b0d992/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T12:45:40,686 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c32549cd7f31e798e52fad0b55b0d992; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60004814, jitterRate=-0.10585859417915344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:40,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c32549cd7f31e798e52fad0b55b0d992: Writing region info on filesystem at 1732538740618Initializing all the Stores at 1732538740619 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740619Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740619Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740619Cleaning up temporary data from old regions at 1732538740682 (+63 ms)Region opened successfully at 1732538740686 (+4 ms) 2024-11-25T12:45:40,689 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c32549cd7f31e798e52fad0b55b0d992, disabling compactions & flushes 2024-11-25T12:45:40,689 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:40,689 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:40,689 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. after waiting 0 ms 2024-11-25T12:45:40,689 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:40,689 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732538736169.c32549cd7f31e798e52fad0b55b0d992. 2024-11-25T12:45:40,690 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c32549cd7f31e798e52fad0b55b0d992: Waiting for close lock at 1732538740689Disabling compacts and flushes for region at 1732538740689Disabling writes for close at 1732538740689Writing region close event to WAL at 1732538740689Closed at 1732538740689 2024-11-25T12:45:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741924_1103 (size=175) 2024-11-25T12:45:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741924_1103 (size=175) 2024-11-25T12:45:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741924_1103 (size=175) 2024-11-25T12:45:40,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:40,694 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538740579) 2024-11-25T12:45:40,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 432) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-712319592_22 at /127.0.0.1:58264 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41185 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-712319592_22 at /127.0.0.1:44146 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-712319592_22 at /127.0.0.1:49832 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=583 (was 616), ProcessCount=11 (was 11), AvailableMemoryMB=1962 (was 1314) - AvailableMemoryMB LEAK? - 2024-11-25T12:45:40,706 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-25T12:45:40,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=583, ProcessCount=11, AvailableMemoryMB=1961 2024-11-25T12:45:40,720 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-25T12:45:40,741 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:40,743 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:45:40,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:45:40,747 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53344725, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/hregion-53344725, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:40,764 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53344725/hregion-53344725.1732538740747, exclude list is [], retry=0 2024-11-25T12:45:40,767 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:40,767 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:40,768 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:40,797 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53344725/hregion-53344725.1732538740747 2024-11-25T12:45:40,798 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:40,798 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b85e21d02072b96cabaf1880dc8596c6, NAME => 'testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41185/hbase 2024-11-25T12:45:40,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741926_1105 (size=67) 2024-11-25T12:45:40,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741926_1105 (size=67) 2024-11-25T12:45:40,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741926_1105 (size=67) 2024-11-25T12:45:40,814 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:40,816 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,817 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName a 2024-11-25T12:45:40,817 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,818 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,818 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,819 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName b 2024-11-25T12:45:40,819 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,820 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,820 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,821 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName c 2024-11-25T12:45:40,821 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,821 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,821 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,822 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,822 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,823 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,823 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,824 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:40,825 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,828 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:45:40,828 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b85e21d02072b96cabaf1880dc8596c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70902129, jitterRate=0.05652405321598053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:40,828 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b85e21d02072b96cabaf1880dc8596c6: Writing region info on filesystem at 1732538740814Initializing all the Stores at 1732538740815 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740815Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740816 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740816Cleaning up temporary data from old regions at 1732538740823 (+7 ms)Region opened successfully at 1732538740828 (+5 ms) 2024-11-25T12:45:40,828 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b85e21d02072b96cabaf1880dc8596c6, disabling compactions & flushes 2024-11-25T12:45:40,828 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:40,828 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:40,828 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. after waiting 0 ms 2024-11-25T12:45:40,829 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:40,836 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:40,836 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b85e21d02072b96cabaf1880dc8596c6: Waiting for close lock at 1732538740828Disabling compacts and flushes for region at 1732538740828Disabling writes for close at 1732538740829 (+1 ms)Writing region close event to WAL at 1732538740836 (+7 ms)Closed at 1732538740836 2024-11-25T12:45:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741925_1104 (size=175) 2024-11-25T12:45:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741925_1104 (size=175) 2024-11-25T12:45:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741925_1104 (size=175) 2024-11-25T12:45:40,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:40,846 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53344725:(num 1732538740747) 2024-11-25T12:45:40,846 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:40,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:40,867 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, exclude list is [], retry=0 2024-11-25T12:45:40,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:40,871 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:40,871 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:40,919 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 2024-11-25T12:45:40,919 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:40,919 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b85e21d02072b96cabaf1880dc8596c6, NAME => 'testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,921 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,922 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName a 2024-11-25T12:45:40,922 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,923 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,923 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,923 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName b 2024-11-25T12:45:40,924 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,924 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,924 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,925 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName c 2024-11-25T12:45:40,925 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:40,925 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:40,925 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,927 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,935 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:40,939 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:40,943 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b85e21d02072b96cabaf1880dc8596c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75324948, jitterRate=0.12242919206619263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:40,944 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b85e21d02072b96cabaf1880dc8596c6: Writing region info on filesystem at 1732538740920Initializing all the Stores at 1732538740921 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740921Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740921Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538740921Cleaning up temporary data from old regions at 1732538740934 (+13 ms)Region opened successfully at 1732538740944 (+10 ms) 2024-11-25T12:45:40,955 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b85e21d02072b96cabaf1880dc8596c6 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T12:45:40,983 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/886cb2acf95743e69f965a14055bc196 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732538740944/Put/seqid=0 2024-11-25T12:45:41,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741928_1107 (size=5958) 2024-11-25T12:45:41,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741928_1107 (size=5958) 2024-11-25T12:45:41,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741928_1107 (size=5958) 2024-11-25T12:45:41,004 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/886cb2acf95743e69f965a14055bc196 2024-11-25T12:45:41,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/886cb2acf95743e69f965a14055bc196 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/886cb2acf95743e69f965a14055bc196 2024-11-25T12:45:41,021 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/886cb2acf95743e69f965a14055bc196, entries=10, sequenceid=13, filesize=5.8 K 2024-11-25T12:45:41,028 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b85e21d02072b96cabaf1880dc8596c6 in 72ms, sequenceid=13, compaction requested=false 2024-11-25T12:45:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b85e21d02072b96cabaf1880dc8596c6: 2024-11-25T12:45:41,057 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b85e21d02072b96cabaf1880dc8596c6, disabling compactions & flushes 2024-11-25T12:45:41,057 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:41,057 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:41,057 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. after waiting 0 ms 2024-11-25T12:45:41,058 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:41,058 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:41,058 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:41,058 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b85e21d02072b96cabaf1880dc8596c6: Waiting for close lock at 1732538741057Disabling compacts and flushes for region at 1732538741057Disabling writes for close at 1732538741058 (+1 ms)Writing region close event to WAL at 1732538741058Closed at 1732538741058 2024-11-25T12:45:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741927_1106 (size=6243) 2024-11-25T12:45:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741927_1106 (size=6243) 2024-11-25T12:45:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741927_1106 (size=6243) 2024-11-25T12:45:41,084 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, size=6.1 K (6243bytes) 2024-11-25T12:45:41,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 2024-11-25T12:45:41,085 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 after 1ms 2024-11-25T12:45:41,117 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:41,117 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 took 33ms 2024-11-25T12:45:41,119 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 so closing down 2024-11-25T12:45:41,119 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:41,120 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732538740851.temp 2024-11-25T12:45:41,147 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp 2024-11-25T12:45:41,148 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:41,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741929_1108 (size=5575) 2024-11-25T12:45:41,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741929_1108 (size=5575) 2024-11-25T12:45:41,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741929_1108 (size=5575) 2024-11-25T12:45:41,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:41,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 2024-11-25T12:45:41,157 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-25T12:45:41,157 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, journal: Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, size=6.1 K (6243bytes) at 1732538741084Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 so closing down at 1732538741119 (+35 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp at 1732538741147 (+28 ms)3 split writer threads finished at 1732538741148 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538741155 (+7 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000003-wal.1732538740851.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 at 1732538741157 (+2 ms)Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732538741157 2024-11-25T12:45:41,159 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538740851 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538740851 2024-11-25T12:45:41,160 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 2024-11-25T12:45:41,160 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:41,162 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:41,174 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, exclude list is [], retry=0 2024-11-25T12:45:41,176 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:41,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:41,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:41,207 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 2024-11-25T12:45:41,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44527:44527),(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585)] 2024-11-25T12:45:41,208 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b85e21d02072b96cabaf1880dc8596c6, NAME => 'testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:45:41,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:41,208 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,208 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,210 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,211 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName a 2024-11-25T12:45:41,211 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:41,217 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/886cb2acf95743e69f965a14055bc196 2024-11-25T12:45:41,217 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:41,217 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,218 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName b 2024-11-25T12:45:41,218 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:41,218 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:41,218 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,219 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName c 2024-11-25T12:45:41,219 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:41,220 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:41,220 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,222 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,222 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 2024-11-25T12:45:41,249 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:41,250 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 2024-11-25T12:45:41,251 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b85e21d02072b96cabaf1880dc8596c6 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-25T12:45:41,264 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d8ac6baa30d14ad58b5ae543eb5cc65e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732538741028/Put/seqid=0 2024-11-25T12:45:41,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741931_1110 (size=5958) 2024-11-25T12:45:41,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741931_1110 (size=5958) 2024-11-25T12:45:41,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741931_1110 (size=5958) 2024-11-25T12:45:41,272 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d8ac6baa30d14ad58b5ae543eb5cc65e 2024-11-25T12:45:41,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/5d1c507d4fde40eab102797626d985b1 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732538741041/Put/seqid=0 2024-11-25T12:45:41,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741932_1111 (size=5958) 2024-11-25T12:45:41,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741932_1111 (size=5958) 2024-11-25T12:45:41,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741932_1111 (size=5958) 2024-11-25T12:45:41,702 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/5d1c507d4fde40eab102797626d985b1 2024-11-25T12:45:41,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d8ac6baa30d14ad58b5ae543eb5cc65e as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/b/d8ac6baa30d14ad58b5ae543eb5cc65e 2024-11-25T12:45:41,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/b/d8ac6baa30d14ad58b5ae543eb5cc65e, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T12:45:41,714 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/5d1c507d4fde40eab102797626d985b1 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/c/5d1c507d4fde40eab102797626d985b1 2024-11-25T12:45:41,719 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/c/5d1c507d4fde40eab102797626d985b1, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T12:45:41,720 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b85e21d02072b96cabaf1880dc8596c6 in 469ms, sequenceid=35, compaction requested=false; wal=null 2024-11-25T12:45:41,720 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000035 2024-11-25T12:45:41,721 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,721 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,722 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:41,723 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:41,725 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-25T12:45:41,726 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b85e21d02072b96cabaf1880dc8596c6; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66683172, jitterRate=-0.0063433051109313965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:41,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b85e21d02072b96cabaf1880dc8596c6: Writing region info on filesystem at 1732538741208Initializing all the Stores at 1732538741209 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538741209Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538741210 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538741210Obtaining lock to block concurrent updates at 1732538741251 (+41 ms)Preparing flush snapshotting stores in b85e21d02072b96cabaf1880dc8596c6 at 1732538741251Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732538741251Flushing stores of testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. at 1732538741251Flushing b85e21d02072b96cabaf1880dc8596c6/b: creating writer at 1732538741251Flushing b85e21d02072b96cabaf1880dc8596c6/b: appending metadata at 1732538741263 (+12 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/b: closing flushed file at 1732538741264 (+1 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/c: creating writer at 1732538741276 (+12 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/c: appending metadata at 1732538741292 (+16 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/c: closing flushed file at 1732538741292Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25b73f81: reopening flushed file at 1732538741708 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2436c48c: reopening flushed file at 1732538741713 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b85e21d02072b96cabaf1880dc8596c6 in 469ms, sequenceid=35, compaction requested=false; wal=null at 1732538741720 (+7 ms)Cleaning up temporary data from old regions at 1732538741721 (+1 ms)Region opened successfully at 1732538741726 (+5 ms) 2024-11-25T12:45:41,801 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, size=0 (0bytes) 2024-11-25T12:45:41,801 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 might be still open, length is 0 2024-11-25T12:45:41,801 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 2024-11-25T12:45:41,802 WARN [IPC Server handler 1 on default port 41185 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-25T12:45:41,804 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 after 3ms 2024-11-25T12:45:42,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:58322 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35695:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58322 dst: /127.0.0.1:35695 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35695 remote=/127.0.0.1:58322]. Total timeout mills is 60000, 58772 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:42,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:44196 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44196 dst: /127.0.0.1:38775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:42,985 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_445138391_22 at /127.0.0.1:49912 [Receiving block BP-322817772-172.17.0.3-1732538693888:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45631:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49912 dst: /127.0.0.1:45631 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:42,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741930_1112 (size=5567) 2024-11-25T12:45:42,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741930_1112 (size=5567) 2024-11-25T12:45:43,995 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T12:45:43,995 INFO [master/1ff8009e463e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T12:45:45,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 after 4010ms 2024-11-25T12:45:45,848 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:45,848 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 took 4047ms 2024-11-25T12:45:45,852 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163; continuing. 2024-11-25T12:45:45,852 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 so closing down 2024-11-25T12:45:45,852 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T12:45:45,853 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732538741163.temp 2024-11-25T12:45:45,909 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp 2024-11-25T12:45:45,909 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T12:45:45,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741933_1113 (size=5575) 2024-11-25T12:45:45,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741933_1113 (size=5575) 2024-11-25T12:45:45,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741933_1113 (size=5575) 2024-11-25T12:45:45,928 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T12:45:45,929 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 2024-11-25T12:45:45,929 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T12:45:45,930 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, journal: Splitting hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, size=0 (0bytes) at 1732538741801Finishing writing output for hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 so closing down at 1732538745852 (+4051 ms)Creating recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp at 1732538745909 (+57 ms)3 split writer threads finished at 1732538745909Closed recovered edits writer path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732538745928 (+19 ms)Rename recovered edits hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000037-wal.1732538741163.temp to hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 at 1732538745929 (+1 ms)Processed 30 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163, size=0, length=0, corrupted=false, cancelled=false at 1732538745929 2024-11-25T12:45:45,931 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 to hdfs://localhost:41185/hbase/oldWALs/wal.1732538741163 2024-11-25T12:45:45,936 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 2024-11-25T12:45:45,936 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T12:45:45,942 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41185/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740, archiveDir=hdfs://localhost:41185/hbase/oldWALs, maxLogs=32 2024-11-25T12:45:45,960 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538745942, exclude list is [], retry=0 2024-11-25T12:45:45,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45631,DS-772d4dd6-0bc9-494f-93a4-955af978e62c,DISK] 2024-11-25T12:45:45,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38775,DS-ed36d016-c388-4fe2-96e7-e4b5fb426dd1,DISK] 2024-11-25T12:45:45,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35695,DS-898c5bdf-bf65-4200-bfe1-c6b18d3dbc8f,DISK] 2024-11-25T12:45:46,007 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538745942 2024-11-25T12:45:46,007 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45865:45865),(127.0.0.1/127.0.0.1:44585:44585),(127.0.0.1/127.0.0.1:44527:44527)] 2024-11-25T12:45:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:45:46,012 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,014 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName a 2024-11-25T12:45:46,014 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:46,024 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/886cb2acf95743e69f965a14055bc196 2024-11-25T12:45:46,024 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:46,024 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,025 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName b 2024-11-25T12:45:46,025 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:46,045 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/b/d8ac6baa30d14ad58b5ae543eb5cc65e 2024-11-25T12:45:46,046 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:46,046 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,047 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b85e21d02072b96cabaf1880dc8596c6 columnFamilyName c 2024-11-25T12:45:46,047 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:45:46,061 DEBUG [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/c/5d1c507d4fde40eab102797626d985b1 2024-11-25T12:45:46,061 INFO [StoreOpener-b85e21d02072b96cabaf1880dc8596c6-1 {}] regionserver.HStore(327): Store=b85e21d02072b96cabaf1880dc8596c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:45:46,061 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,064 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 2024-11-25T12:45:46,105 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T12:45:46,112 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 2024-11-25T12:45:46,112 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b85e21d02072b96cabaf1880dc8596c6 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T12:45:46,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/4d3a9011cda242eb9a519690159c9f85 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732538741734/Put/seqid=0 2024-11-25T12:45:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741935_1115 (size=5958) 2024-11-25T12:45:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741935_1115 (size=5958) 2024-11-25T12:45:46,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741935_1115 (size=5958) 2024-11-25T12:45:46,177 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/4d3a9011cda242eb9a519690159c9f85 2024-11-25T12:45:46,217 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d17e3e88317244428131fb65f605bfc4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732538741742/Put/seqid=0 2024-11-25T12:45:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741936_1116 (size=5958) 2024-11-25T12:45:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741936_1116 (size=5958) 2024-11-25T12:45:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741936_1116 (size=5958) 2024-11-25T12:45:46,247 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d17e3e88317244428131fb65f605bfc4 2024-11-25T12:45:46,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/52f3446787464687a3caf3e4745da7af is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732538741750/Put/seqid=0 2024-11-25T12:45:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741937_1117 (size=5958) 2024-11-25T12:45:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741937_1117 (size=5958) 2024-11-25T12:45:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741937_1117 (size=5958) 2024-11-25T12:45:46,274 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/52f3446787464687a3caf3e4745da7af 2024-11-25T12:45:46,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/a/4d3a9011cda242eb9a519690159c9f85 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/4d3a9011cda242eb9a519690159c9f85 2024-11-25T12:45:46,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/a/4d3a9011cda242eb9a519690159c9f85, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:45:46,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/b/d17e3e88317244428131fb65f605bfc4 as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/b/d17e3e88317244428131fb65f605bfc4 2024-11-25T12:45:46,288 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/b/d17e3e88317244428131fb65f605bfc4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:45:46,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/.tmp/c/52f3446787464687a3caf3e4745da7af as hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/c/52f3446787464687a3caf3e4745da7af 2024-11-25T12:45:46,293 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/c/52f3446787464687a3caf3e4745da7af, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T12:45:46,293 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b85e21d02072b96cabaf1880dc8596c6 in 181ms, sequenceid=66, compaction requested=false; wal=null 2024-11-25T12:45:46,294 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/0000000000000000066 2024-11-25T12:45:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,295 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T12:45:46,296 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b85e21d02072b96cabaf1880dc8596c6 2024-11-25T12:45:46,298 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/hbase/data/default/testReplayEditsWrittenViaHRegion/b85e21d02072b96cabaf1880dc8596c6/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-25T12:45:46,299 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b85e21d02072b96cabaf1880dc8596c6; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66511105, jitterRate=-0.008907303214073181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T12:45:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b85e21d02072b96cabaf1880dc8596c6: Writing region info on filesystem at 1732538746008Initializing all the Stores at 1732538746009 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538746009Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538746012 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732538746012Obtaining lock to block concurrent updates at 1732538746112 (+100 ms)Preparing flush snapshotting stores in b85e21d02072b96cabaf1880dc8596c6 at 1732538746112Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732538746112Flushing stores of testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. at 1732538746112Flushing b85e21d02072b96cabaf1880dc8596c6/a: creating writer at 1732538746112Flushing b85e21d02072b96cabaf1880dc8596c6/a: appending metadata at 1732538746140 (+28 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/a: closing flushed file at 1732538746141 (+1 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/b: creating writer at 1732538746188 (+47 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/b: appending metadata at 1732538746217 (+29 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/b: closing flushed file at 1732538746217Flushing b85e21d02072b96cabaf1880dc8596c6/c: creating writer at 1732538746253 (+36 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/c: appending metadata at 1732538746267 (+14 ms)Flushing b85e21d02072b96cabaf1880dc8596c6/c: closing flushed file at 1732538746267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bb41e0d: reopening flushed file at 1732538746278 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195ecc2e: reopening flushed file at 1732538746283 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f363b81: reopening flushed file at 1732538746288 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b85e21d02072b96cabaf1880dc8596c6 in 181ms, sequenceid=66, compaction requested=false; wal=null at 1732538746293 (+5 ms)Cleaning up temporary data from old regions at 1732538746295 (+2 ms)Region opened successfully at 1732538746299 (+4 ms) 2024-11-25T12:45:46,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b85e21d02072b96cabaf1880dc8596c6, disabling compactions & flushes 2024-11-25T12:45:46,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:46,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:46,311 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. after waiting 0 ms 2024-11-25T12:45:46,311 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:46,313 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732538740741.b85e21d02072b96cabaf1880dc8596c6. 2024-11-25T12:45:46,313 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b85e21d02072b96cabaf1880dc8596c6: Waiting for close lock at 1732538746311Disabling compacts and flushes for region at 1732538746311Disabling writes for close at 1732538746311Writing region close event to WAL at 1732538746313 (+2 ms)Closed at 1732538746313 2024-11-25T12:45:46,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741934_1114 (size=175) 2024-11-25T12:45:46,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741934_1114 (size=175) 2024-11-25T12:45:46,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741934_1114 (size=175) 2024-11-25T12:45:46,320 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T12:45:46,320 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732538745942) 2024-11-25T12:45:46,336 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=447 (was 443) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1684072433_22 at /127.0.0.1:58792 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41185 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1684072433_22 at /127.0.0.1:37370 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1684072433_22 at /127.0.0.1:58116 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:41185 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 583), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=2551 (was 1961) - AvailableMemoryMB LEAK? - 2024-11-25T12:45:46,336 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-25T12:45:46,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T12:45:46,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:45:46,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:45:46,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:45:46,338 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T12:45:46,338 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1125179072, stopped=false 2024-11-25T12:45:46,338 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ff8009e463e,43183,1732538699066 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:46,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:46,356 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:45:46,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:46,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:46,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:45:46,357 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:45:46,357 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:45:46,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,358 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,35011,1732538700146' ***** 2024-11-25T12:45:46,358 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T12:45:46,358 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ff8009e463e,44737,1732538700372' ***** 2024-11-25T12:45:46,358 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T12:45:46,358 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:45:46,358 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:45:46,358 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:45:46,358 INFO [RS:0;1ff8009e463e:35011 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T12:45:46,358 INFO [RS:0;1ff8009e463e:35011 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T12:45:46,358 INFO [RS:2;1ff8009e463e:44737 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T12:45:46,358 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:45:46,358 INFO [RS:2;1ff8009e463e:44737 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T12:45:46,358 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(959): stopping server 1ff8009e463e,35011,1732538700146 2024-11-25T12:45:46,359 INFO [RS:0;1ff8009e463e:35011 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:45:46,359 INFO [RS:0;1ff8009e463e:35011 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ff8009e463e:35011. 2024-11-25T12:45:46,359 DEBUG [RS:0;1ff8009e463e:35011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(3091): Received CLOSE for 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:46,359 DEBUG [RS:0;1ff8009e463e:35011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,359 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,35011,1732538700146; all regions closed. 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(959): stopping server 1ff8009e463e,44737,1732538700372 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1ff8009e463e:44737. 2024-11-25T12:45:46,359 DEBUG [RS:2;1ff8009e463e:44737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:45:46,359 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88bc9a72f668c659e38c0669352cfc0c, disabling compactions & flushes 2024-11-25T12:45:46,359 DEBUG [RS:2;1ff8009e463e:44737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,359 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:45:46,359 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:45:46,359 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. after waiting 0 ms 2024-11-25T12:45:46,359 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:45:46,360 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:46,360 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T12:45:46,360 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T12:45:46,360 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 88bc9a72f668c659e38c0669352cfc0c=testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c.} 2024-11-25T12:45:46,360 DEBUG [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 88bc9a72f668c659e38c0669352cfc0c 2024-11-25T12:45:46,360 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:45:46,360 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:45:46,360 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:45:46,360 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:45:46,360 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:45:46,360 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-25T12:45:46,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741835_1011 (size=2764) 2024-11-25T12:45:46,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741835_1011 (size=2764) 2024-11-25T12:45:46,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741835_1011 (size=2764) 2024-11-25T12:45:46,368 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88bc9a72f668c659e38c0669352cfc0c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-25T12:45:46,368 DEBUG [RS:0;1ff8009e463e:35011 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C35011%2C1732538700146:(num 1732538702564) 2024-11-25T12:45:46,368 DEBUG [RS:0;1ff8009e463e:35011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:45:46,368 INFO [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T12:45:46,368 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88bc9a72f668c659e38c0669352cfc0c: Waiting for close lock at 1732538746359Running coprocessor pre-close hooks at 1732538746359Disabling compacts and flushes for region at 1732538746359Disabling writes for close at 1732538746359Writing region close event to WAL at 1732538746363 (+4 ms)Running coprocessor post-close hooks at 1732538746368 (+5 ms)Closed at 1732538746368 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:45:46,368 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:45:46,368 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:45:46,369 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:45:46,369 INFO [RS:0;1ff8009e463e:35011 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:45:46,369 DEBUG [RS_CLOSE_REGION-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c. 2024-11-25T12:45:46,369 INFO [RS:0;1ff8009e463e:35011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35011 2024-11-25T12:45:46,385 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:46,388 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/info/356845d63b324068ab2841e9d85add11 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732538720178.88bc9a72f668c659e38c0669352cfc0c./info:regioninfo/1732538724220/Put/seqid=0 2024-11-25T12:45:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741938_1118 (size=8243) 2024-11-25T12:45:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741938_1118 (size=8243) 2024-11-25T12:45:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741938_1118 (size=8243) 2024-11-25T12:45:46,395 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/info/356845d63b324068ab2841e9d85add11 2024-11-25T12:45:46,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,35011,1732538700146 2024-11-25T12:45:46,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:46,411 INFO [RS:0;1ff8009e463e:35011 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:45:46,421 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/ns/f17b7a020e824a448098ebef6945e914 is 43, key is default/ns:d/1732538703265/Put/seqid=0 2024-11-25T12:45:46,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741939_1119 (size=5153) 2024-11-25T12:45:46,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741939_1119 (size=5153) 2024-11-25T12:45:46,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741939_1119 (size=5153) 2024-11-25T12:45:46,427 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/ns/f17b7a020e824a448098ebef6945e914 2024-11-25T12:45:46,431 INFO [regionserver/1ff8009e463e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:46,439 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,35011,1732538700146] 2024-11-25T12:45:46,446 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/table/3af94fe6aaf848efa8b8c67ddfcf12f8 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732538720788/Put/seqid=0 2024-11-25T12:45:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741940_1120 (size=5431) 2024-11-25T12:45:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741940_1120 (size=5431) 2024-11-25T12:45:46,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741940_1120 (size=5431) 2024-11-25T12:45:46,452 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/table/3af94fe6aaf848efa8b8c67ddfcf12f8 2024-11-25T12:45:46,460 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/info/356845d63b324068ab2841e9d85add11 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/info/356845d63b324068ab2841e9d85add11 2024-11-25T12:45:46,464 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,35011,1732538700146 already deleted, retry=false 2024-11-25T12:45:46,464 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ff8009e463e,35011,1732538700146 expired; onlineServers=1 2024-11-25T12:45:46,465 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/info/356845d63b324068ab2841e9d85add11, entries=18, sequenceid=21, filesize=8.0 K 2024-11-25T12:45:46,466 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/ns/f17b7a020e824a448098ebef6945e914 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/ns/f17b7a020e824a448098ebef6945e914 2024-11-25T12:45:46,471 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/ns/f17b7a020e824a448098ebef6945e914, entries=2, sequenceid=21, filesize=5.0 K 2024-11-25T12:45:46,472 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/.tmp/table/3af94fe6aaf848efa8b8c67ddfcf12f8 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/table/3af94fe6aaf848efa8b8c67ddfcf12f8 2024-11-25T12:45:46,476 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/table/3af94fe6aaf848efa8b8c67ddfcf12f8, entries=2, sequenceid=21, filesize=5.3 K 2024-11-25T12:45:46,477 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=21, compaction requested=false 2024-11-25T12:45:46,481 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-25T12:45:46,482 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:45:46,482 INFO [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:45:46,482 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732538746360Running coprocessor pre-close hooks at 1732538746360Disabling compacts and flushes for region at 1732538746360Disabling writes for close at 1732538746360Obtaining lock to block concurrent updates at 1732538746361 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732538746361Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732538746361Flushing stores of hbase:meta,,1.1588230740 at 1732538746363 (+2 ms)Flushing 1588230740/info: creating writer at 1732538746363Flushing 1588230740/info: appending metadata at 1732538746388 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732538746388Flushing 1588230740/ns: creating writer at 1732538746401 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732538746420 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732538746420Flushing 1588230740/table: creating writer at 1732538746432 (+12 ms)Flushing 1588230740/table: appending metadata at 1732538746445 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732538746445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b93be8c: reopening flushed file at 1732538746459 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@482442dc: reopening flushed file at 1732538746465 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c29d02: reopening flushed file at 1732538746471 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=21, compaction requested=false at 1732538746477 (+6 ms)Writing region close event to WAL at 1732538746478 (+1 ms)Running coprocessor post-close hooks at 1732538746482 (+4 ms)Closed at 1732538746482 2024-11-25T12:45:46,482 DEBUG [RS_CLOSE_META-regionserver/1ff8009e463e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T12:45:46,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:46,539 INFO [RS:0;1ff8009e463e:35011 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:45:46,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35011-0x10173503b590001, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:46,539 INFO [RS:0;1ff8009e463e:35011 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,35011,1732538700146; zookeeper connection closed. 2024-11-25T12:45:46,540 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5820aaab {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5820aaab 2024-11-25T12:45:46,560 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(976): stopping server 1ff8009e463e,44737,1732538700372; all regions closed. 2024-11-25T12:45:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741836_1012 (size=9487) 2024-11-25T12:45:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741836_1012 (size=9487) 2024-11-25T12:45:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741836_1012 (size=9487) 2024-11-25T12:45:46,572 DEBUG [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs 2024-11-25T12:45:46,572 INFO [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C44737%2C1732538700372.meta:.meta(num 1732538703034) 2024-11-25T12:45:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741833_1009 (size=1032) 2024-11-25T12:45:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741833_1009 (size=1032) 2024-11-25T12:45:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741833_1009 (size=1032) 2024-11-25T12:45:46,576 DEBUG [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/oldWALs 2024-11-25T12:45:46,576 INFO [RS:2;1ff8009e463e:44737 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ff8009e463e%2C44737%2C1732538700372:(num 1732538702496) 2024-11-25T12:45:46,576 DEBUG [RS:2;1ff8009e463e:44737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:45:46,576 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:45:46,577 INFO [RS:2;1ff8009e463e:44737 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:45:46,577 INFO [RS:2;1ff8009e463e:44737 {}] hbase.ChoreService(370): Chore service for: regionserver/1ff8009e463e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T12:45:46,577 INFO [RS:2;1ff8009e463e:44737 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:45:46,577 INFO [regionserver/1ff8009e463e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:45:46,577 INFO [RS:2;1ff8009e463e:44737 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44737 2024-11-25T12:45:46,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:45:46,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ff8009e463e,44737,1732538700372 2024-11-25T12:45:46,594 INFO [RS:2;1ff8009e463e:44737 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:45:46,595 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ff8009e463e,44737,1732538700372] 2024-11-25T12:45:46,652 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ff8009e463e,44737,1732538700372 already deleted, retry=false 2024-11-25T12:45:46,652 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ff8009e463e,44737,1732538700372 expired; onlineServers=0 2024-11-25T12:45:46,652 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ff8009e463e,43183,1732538699066' ***** 2024-11-25T12:45:46,653 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T12:45:46,653 INFO [M:0;1ff8009e463e:43183 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:45:46,653 INFO [M:0;1ff8009e463e:43183 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:45:46,653 DEBUG [M:0;1ff8009e463e:43183 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T12:45:46,653 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T12:45:46,653 DEBUG [M:0;1ff8009e463e:43183 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T12:45:46,653 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538702102 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.large.0-1732538702102,5,FailOnTimeoutGroup] 2024-11-25T12:45:46,653 DEBUG [master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538702103 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ff8009e463e:0:becomeActiveMaster-HFileCleaner.small.0-1732538702103,5,FailOnTimeoutGroup] 2024-11-25T12:45:46,653 INFO [M:0;1ff8009e463e:43183 {}] hbase.ChoreService(370): Chore service for: master/1ff8009e463e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T12:45:46,654 INFO [M:0;1ff8009e463e:43183 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:45:46,654 DEBUG [M:0;1ff8009e463e:43183 {}] master.HMaster(1795): Stopping service threads 2024-11-25T12:45:46,654 INFO [M:0;1ff8009e463e:43183 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T12:45:46,654 INFO [M:0;1ff8009e463e:43183 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:45:46,655 INFO [M:0;1ff8009e463e:43183 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T12:45:46,655 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T12:45:46,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T12:45:46,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:45:46,667 DEBUG [M:0;1ff8009e463e:43183 {}] zookeeper.ZKUtil(347): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T12:45:46,667 WARN [M:0;1ff8009e463e:43183 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T12:45:46,668 INFO [M:0;1ff8009e463e:43183 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/.lastflushedseqids 2024-11-25T12:45:46,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741941_1121 (size=138) 2024-11-25T12:45:46,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741941_1121 (size=138) 2024-11-25T12:45:46,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741941_1121 (size=138) 2024-11-25T12:45:46,688 INFO [M:0;1ff8009e463e:43183 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T12:45:46,688 INFO [M:0;1ff8009e463e:43183 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T12:45:46,688 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:45:46,688 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:46,688 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:46,688 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:45:46,688 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:46,688 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-25T12:45:46,706 DEBUG [M:0;1ff8009e463e:43183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac70109b68944ad681474252d51ce46a is 82, key is hbase:meta,,1/info:regioninfo/1732538703175/Put/seqid=0 2024-11-25T12:45:46,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:46,723 INFO [RS:2;1ff8009e463e:44737 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:45:46,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44737-0x10173503b590003, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:46,723 INFO [RS:2;1ff8009e463e:44737 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ff8009e463e,44737,1732538700372; zookeeper connection closed. 2024-11-25T12:45:46,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741942_1122 (size=5672) 2024-11-25T12:45:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741942_1122 (size=5672) 2024-11-25T12:45:46,724 INFO [M:0;1ff8009e463e:43183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac70109b68944ad681474252d51ce46a 2024-11-25T12:45:46,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741942_1122 (size=5672) 2024-11-25T12:45:46,731 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10448c73 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10448c73 2024-11-25T12:45:46,732 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T12:45:46,754 DEBUG [M:0;1ff8009e463e:43183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7930afad89b4081bc75a5133737911c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732538720797/Put/seqid=0 2024-11-25T12:45:46,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741943_1123 (size=7755) 2024-11-25T12:45:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741943_1123 (size=7755) 2024-11-25T12:45:46,810 INFO [M:0;1ff8009e463e:43183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7930afad89b4081bc75a5133737911c 2024-11-25T12:45:46,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741943_1123 (size=7755) 2024-11-25T12:45:46,823 INFO [M:0;1ff8009e463e:43183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d7930afad89b4081bc75a5133737911c 2024-11-25T12:45:46,846 DEBUG [M:0;1ff8009e463e:43183 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ead8e597bd2142b5aa861aba204a5dc8 is 69, key is 1ff8009e463e,35011,1732538700146/rs:state/1732538702296/Put/seqid=0 2024-11-25T12:45:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741944_1124 (size=5445) 2024-11-25T12:45:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741944_1124 (size=5445) 2024-11-25T12:45:46,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741944_1124 (size=5445) 2024-11-25T12:45:46,866 INFO [M:0;1ff8009e463e:43183 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ead8e597bd2142b5aa861aba204a5dc8 2024-11-25T12:45:46,873 INFO [M:0;1ff8009e463e:43183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ead8e597bd2142b5aa861aba204a5dc8 2024-11-25T12:45:46,876 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ac70109b68944ad681474252d51ce46a as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ac70109b68944ad681474252d51ce46a 2024-11-25T12:45:46,885 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ac70109b68944ad681474252d51ce46a, entries=8, sequenceid=168, filesize=5.5 K 2024-11-25T12:45:46,886 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d7930afad89b4081bc75a5133737911c as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d7930afad89b4081bc75a5133737911c 2024-11-25T12:45:46,896 INFO [M:0;1ff8009e463e:43183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d7930afad89b4081bc75a5133737911c 2024-11-25T12:45:46,896 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d7930afad89b4081bc75a5133737911c, entries=17, sequenceid=168, filesize=7.6 K 2024-11-25T12:45:46,898 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ead8e597bd2142b5aa861aba204a5dc8 as hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ead8e597bd2142b5aa861aba204a5dc8 2024-11-25T12:45:46,905 INFO [M:0;1ff8009e463e:43183 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ead8e597bd2142b5aa861aba204a5dc8 2024-11-25T12:45:46,905 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41185/user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ead8e597bd2142b5aa861aba204a5dc8, entries=3, sequenceid=168, filesize=5.3 K 2024-11-25T12:45:46,906 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 218ms, sequenceid=168, compaction requested=false 2024-11-25T12:45:46,912 INFO [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:45:46,912 DEBUG [M:0;1ff8009e463e:43183 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732538746688Disabling compacts and flushes for region at 1732538746688Disabling writes for close at 1732538746688Obtaining lock to block concurrent updates at 1732538746688Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732538746688Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732538746689 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732538746689Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732538746689Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732538746705 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732538746705Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732538746732 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732538746753 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732538746753Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732538746823 (+70 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732538746845 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732538746845Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19319d0a: reopening flushed file at 1732538746873 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3989e78f: reopening flushed file at 1732538746885 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d78d051: reopening flushed file at 1732538746897 (+12 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 218ms, sequenceid=168, compaction requested=false at 1732538746906 (+9 ms)Writing region close event to WAL at 1732538746912 (+6 ms)Closed at 1732538746912 2024-11-25T12:45:46,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38775 is added to blk_1073741830_1006 (size=83606) 2024-11-25T12:45:46,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35695 is added to blk_1073741830_1006 (size=83606) 2024-11-25T12:45:46,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45631 is added to blk_1073741830_1006 (size=83606) 2024-11-25T12:45:46,920 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/33f4ed7f-847a-1709-919f-23dfaa3979c3/MasterData/WALs/1ff8009e463e,43183,1732538699066/1ff8009e463e%2C43183%2C1732538699066.1732538701115 not finished, retry = 0 2024-11-25T12:45:47,022 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:45:47,022 INFO [M:0;1ff8009e463e:43183 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T12:45:47,023 INFO [M:0;1ff8009e463e:43183 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43183 2024-11-25T12:45:47,023 INFO [M:0;1ff8009e463e:43183 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:45:47,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:47,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43183-0x10173503b590000, quorum=127.0.0.1:59707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:45:47,136 INFO [M:0;1ff8009e463e:43183 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:45:47,159 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538725319 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538725319 (inode 16655) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732538724506/wal.1732538725319 (inode 16655) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:45:47,169 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732538715336/wal.1732538715478 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:47,169 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732538740740/wal.1732538741163 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:47,180 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538735061 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538735061 (inode 16678) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732538725667/wal.1732538735061 (inode 16678) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:45:47,181 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732538705739/wal.1732538705921 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:47,192 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735978 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735978 (inode 16704) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732538735347/wal.1732538735978 (inode 16704) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:45:47,194 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732538736167/wal.1732538736255 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:45:47,212 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal.1732538705495 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal.1732538705495 (inode 16485) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732538704878/wal.1732538705495 (inode 16485) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:45:47,226 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal.1732538704422 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal.1732538704422 (inode 16462) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732538703675/wal.1732538704422 (inode 16462) Holder DFSClient_NONMAPREDUCE_445138391_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-25T12:45:47,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:47,243 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:45:47,243 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:45:47,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:45:47,244 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,STOPPED} 2024-11-25T12:45:47,252 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:45:47,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:45:47,252 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-322817772-172.17.0.3-1732538693888 (Datanode Uuid 135a58b5-b6c0-4766-9725-2aa3aa805de0) service to localhost/127.0.0.1:41185 2024-11-25T12:45:47,252 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:45:47,254 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data5/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,256 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:45:47,257 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data6/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:47,287 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:45:47,288 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:45:47,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:45:47,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,STOPPED} 2024-11-25T12:45:47,292 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:45:47,292 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:45:47,295 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:45:47,295 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-322817772-172.17.0.3-1732538693888 (Datanode Uuid 2d68da22-7be8-4f1a-9926-dc2b6466d4c2) service to localhost/127.0.0.1:41185 2024-11-25T12:45:47,297 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data4/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,297 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:45:47,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data3/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:45:47,308 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:45:47,308 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:45:47,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:45:47,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,STOPPED} 2024-11-25T12:45:47,315 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:45:47,315 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:45:47,315 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:45:47,315 WARN [BP-322817772-172.17.0.3-1732538693888 heartbeating to localhost/127.0.0.1:41185 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-322817772-172.17.0.3-1732538693888 (Datanode Uuid 8a0479a1-41ed-4b63-bc49-f178a653752d) service to localhost/127.0.0.1:41185 2024-11-25T12:45:47,316 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data1/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,317 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:45:47,319 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/cluster_26846607-e206-9bdb-8493-14b863b4d718/data/data2/current/BP-322817772-172.17.0.3-1732538693888 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:45:47,333 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:45:47,334 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:45:47,334 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:45:47,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:45:47,335 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/127e1035-95aa-06a4-633a-2f493e3b11c6/hadoop.log.dir/,STOPPED} 2024-11-25T12:45:47,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T12:45:47,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down