2024-11-13 03:32:28,694 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 03:32:28,704 main DEBUG Took 0.008918 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 03:32:28,705 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 03:32:28,705 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 03:32:28,706 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 03:32:28,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,714 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 03:32:28,725 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,727 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,728 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,730 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,731 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,732 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,733 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,733 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,734 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,736 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,736 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:32:28,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,737 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 03:32:28,739 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:32:28,740 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 03:32:28,742 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 03:32:28,742 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 03:32:28,743 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 03:32:28,744 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 03:32:28,751 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 03:32:28,754 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 03:32:28,755 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 03:32:28,756 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 03:32:28,756 main DEBUG createAppenders(={Console}) 2024-11-13 03:32:28,757 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-13 03:32:28,757 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 03:32:28,757 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-13 03:32:28,758 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 03:32:28,758 main DEBUG OutputStream closed 2024-11-13 03:32:28,758 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 03:32:28,758 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 03:32:28,759 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-13 03:32:28,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 03:32:28,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 03:32:28,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 03:32:28,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 03:32:28,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 03:32:28,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 03:32:28,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 03:32:28,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 03:32:28,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 03:32:28,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 03:32:28,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 03:32:28,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 03:32:28,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 03:32:28,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 03:32:28,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 03:32:28,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 03:32:28,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 03:32:28,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 03:32:28,836 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 03:32:28,836 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-13 03:32:28,837 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 03:32:28,837 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-13T03:32:29,087 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131 2024-11-13 03:32:29,090 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 03:32:29,090 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T03:32:29,098 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T03:32:29,105 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-13T03:32:29,128 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T03:32:29,171 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T03:32:29,171 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T03:32:29,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T03:32:29,198 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351, deleteOnExit=true 2024-11-13T03:32:29,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T03:32:29,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/test.cache.data in system properties and HBase conf 2024-11-13T03:32:29,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T03:32:29,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir in system properties and HBase conf 2024-11-13T03:32:29,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T03:32:29,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T03:32:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T03:32:29,275 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T03:32:29,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T03:32:29,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:32:29,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:32:29,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T03:32:29,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:32:29,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T03:32:29,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T03:32:29,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:32:29,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:32:29,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T03:32:29,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/nfs.dump.dir in system properties and HBase conf 2024-11-13T03:32:29,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/java.io.tmpdir in system properties and HBase conf 2024-11-13T03:32:29,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:32:29,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T03:32:29,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T03:32:30,917 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T03:32:30,989 INFO [Time-limited test {}] log.Log(170): Logging initialized @2985ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T03:32:31,065 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:32:31,129 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:32:31,151 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:32:31,151 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:32:31,153 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:32:31,167 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:32:31,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:32:31,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:32:31,376 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/java.io.tmpdir/jetty-localhost-44507-hadoop-hdfs-3_4_1-tests_jar-_-any-13449346077149292026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:32:31,381 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:44507} 2024-11-13T03:32:31,381 INFO [Time-limited test {}] server.Server(415): Started @3378ms 2024-11-13T03:32:31,914 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:32:31,921 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:32:31,922 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:32:31,922 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:32:31,922 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:32:31,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:32:31,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:32:32,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/java.io.tmpdir/jetty-localhost-42409-hadoop-hdfs-3_4_1-tests_jar-_-any-15425267868958384721/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:32,024 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:42409} 2024-11-13T03:32:32,024 INFO [Time-limited test {}] server.Server(415): Started @4021ms 2024-11-13T03:32:32,076 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:32:32,186 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:32:32,194 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:32:32,196 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:32:32,196 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:32:32,196 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:32:32,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:32:32,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:32:32,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/java.io.tmpdir/jetty-localhost-46281-hadoop-hdfs-3_4_1-tests_jar-_-any-2896175814961428333/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:32,302 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:46281} 2024-11-13T03:32:32,302 INFO [Time-limited test {}] server.Server(415): Started @4299ms 2024-11-13T03:32:32,304 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:32:32,337 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:32:32,342 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:32:32,343 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:32:32,344 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:32:32,344 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:32:32,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:32:32,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:32:32,487 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/java.io.tmpdir/jetty-localhost-40133-hadoop-hdfs-3_4_1-tests_jar-_-any-9919695862579905605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:32,488 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:40133} 2024-11-13T03:32:32,488 INFO [Time-limited test {}] server.Server(415): Started @4485ms 2024-11-13T03:32:32,490 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:32:33,881 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data3/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,881 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data4/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,881 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data1/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,881 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data5/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,881 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data6/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,881 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data2/current/BP-307318758-172.17.0.3-1731468749903/current, will proceed with Du for space computation calculation, 2024-11-13T03:32:33,917 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:32:33,917 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:32:33,917 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:32:33,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf26c014a0dc84a9d with lease ID 0xf561df2e47780363: Processing first storage report for DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4 from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=69c4d9cd-316e-41f2-b74b-20110b5622c2, infoPort=43357, infoSecurePort=0, ipcPort=36277, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf26c014a0dc84a9d with lease ID 0xf561df2e47780363: from storage DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4 node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=69c4d9cd-316e-41f2-b74b-20110b5622c2, infoPort=43357, infoSecurePort=0, ipcPort=36277, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T03:32:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51a7e5d7eca3052e with lease ID 0xf561df2e47780362: Processing first storage report for DS-5650327c-e752-4068-a9e4-0ba1ca837d86 from datanode DatanodeRegistration(127.0.0.1:36333, datanodeUuid=f905237a-6464-4985-a710-f3ea5d90bb18, infoPort=37535, infoSecurePort=0, ipcPort=33081, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51a7e5d7eca3052e with lease ID 0xf561df2e47780362: from storage DS-5650327c-e752-4068-a9e4-0ba1ca837d86 node DatanodeRegistration(127.0.0.1:36333, datanodeUuid=f905237a-6464-4985-a710-f3ea5d90bb18, infoPort=37535, infoSecurePort=0, ipcPort=33081, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:32:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xce12a67e5a7b5ba9 with lease ID 0xf561df2e47780361: Processing first storage report for DS-889197dd-371a-409e-bdeb-c46bc227082b from datanode DatanodeRegistration(127.0.0.1:41693, datanodeUuid=5d84471c-1613-4e35-ac39-357dec741b01, infoPort=42365, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce12a67e5a7b5ba9 with lease ID 0xf561df2e47780361: from storage DS-889197dd-371a-409e-bdeb-c46bc227082b node DatanodeRegistration(127.0.0.1:41693, datanodeUuid=5d84471c-1613-4e35-ac39-357dec741b01, infoPort=42365, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:32:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf26c014a0dc84a9d with lease ID 0xf561df2e47780363: Processing first storage report for DS-bba0dc43-4e5e-4bae-a4d8-0180691dce9d from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=69c4d9cd-316e-41f2-b74b-20110b5622c2, infoPort=43357, infoSecurePort=0, ipcPort=36277, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf26c014a0dc84a9d with lease ID 0xf561df2e47780363: from storage DS-bba0dc43-4e5e-4bae-a4d8-0180691dce9d node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=69c4d9cd-316e-41f2-b74b-20110b5622c2, infoPort=43357, infoSecurePort=0, ipcPort=36277, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:32:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51a7e5d7eca3052e with lease ID 0xf561df2e47780362: Processing first storage report for DS-507481b2-80d4-42b6-9b19-9f30b4d2c634 from datanode DatanodeRegistration(127.0.0.1:36333, datanodeUuid=f905237a-6464-4985-a710-f3ea5d90bb18, infoPort=37535, infoSecurePort=0, ipcPort=33081, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51a7e5d7eca3052e with lease ID 0xf561df2e47780362: from storage DS-507481b2-80d4-42b6-9b19-9f30b4d2c634 node DatanodeRegistration(127.0.0.1:36333, datanodeUuid=f905237a-6464-4985-a710-f3ea5d90bb18, infoPort=37535, infoSecurePort=0, ipcPort=33081, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:32:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xce12a67e5a7b5ba9 with lease ID 0xf561df2e47780361: Processing first storage report for DS-3ee2a2af-9675-4340-9d85-66cd1794a009 from datanode DatanodeRegistration(127.0.0.1:41693, datanodeUuid=5d84471c-1613-4e35-ac39-357dec741b01, infoPort=42365, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903) 2024-11-13T03:32:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce12a67e5a7b5ba9 with lease ID 0xf561df2e47780361: from storage DS-3ee2a2af-9675-4340-9d85-66cd1794a009 node DatanodeRegistration(127.0.0.1:41693, datanodeUuid=5d84471c-1613-4e35-ac39-357dec741b01, infoPort=42365, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1317858723;c=1731468749903), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T03:32:34,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131 2024-11-13T03:32:34,090 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/zookeeper_0, clientPort=56173, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T03:32:34,098 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56173 2024-11-13T03:32:34,107 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:34,109 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:34,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:32:34,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:32:34,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:32:34,710 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed with version=8 2024-11-13T03:32:34,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/hbase-staging 2024-11-13T03:32:34,984 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:32:34,992 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:34,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:34,997 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:32:34,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:34,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:32:35,125 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T03:32:35,176 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T03:32:35,184 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T03:32:35,188 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:32:35,210 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 91955 (auto-detected) 2024-11-13T03:32:35,211 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T03:32:35,230 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38791 2024-11-13T03:32:35,250 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38791 connecting to ZooKeeper ensemble=127.0.0.1:56173 2024-11-13T03:32:35,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387910x0, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:32:35,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38791-0x101324078460000 connected 2024-11-13T03:32:35,484 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,489 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,500 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:35,503 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed, hbase.cluster.distributed=false 2024-11-13T03:32:35,524 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:32:35,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38791 2024-11-13T03:32:35,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38791 2024-11-13T03:32:35,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38791 2024-11-13T03:32:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38791 2024-11-13T03:32:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38791 2024-11-13T03:32:35,632 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:32:35,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,634 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:32:35,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:32:35,637 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:32:35,640 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:32:35,641 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38391 2024-11-13T03:32:35,643 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38391 connecting to ZooKeeper ensemble=127.0.0.1:56173 2024-11-13T03:32:35,644 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,647 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383910x0, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:32:35,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:383910x0, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:35,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38391-0x101324078460001 connected 2024-11-13T03:32:35,665 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:32:35,673 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:32:35,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:32:35,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:32:35,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38391 2024-11-13T03:32:35,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38391 2024-11-13T03:32:35,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38391 2024-11-13T03:32:35,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38391 2024-11-13T03:32:35,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38391 2024-11-13T03:32:35,704 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:32:35,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,705 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:32:35,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:32:35,705 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:32:35,705 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:32:35,706 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35657 2024-11-13T03:32:35,708 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35657 connecting to ZooKeeper ensemble=127.0.0.1:56173 2024-11-13T03:32:35,709 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,712 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:356570x0, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:32:35,724 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:356570x0, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:35,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35657-0x101324078460002 connected 2024-11-13T03:32:35,725 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:32:35,726 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:32:35,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:32:35,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:32:35,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35657 2024-11-13T03:32:35,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35657 2024-11-13T03:32:35,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35657 2024-11-13T03:32:35,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35657 2024-11-13T03:32:35,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35657 2024-11-13T03:32:35,751 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:32:35,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,752 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:32:35,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:32:35,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:32:35,753 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:32:35,753 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:32:35,754 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46881 2024-11-13T03:32:35,757 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46881 connecting to ZooKeeper ensemble=127.0.0.1:56173 2024-11-13T03:32:35,758 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,761 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468810x0, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:32:35,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468810x0, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:35,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46881-0x101324078460003 connected 2024-11-13T03:32:35,777 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:32:35,778 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:32:35,779 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:32:35,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:32:35,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-13T03:32:35,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46881 2024-11-13T03:32:35,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46881 2024-11-13T03:32:35,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-13T03:32:35,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-13T03:32:35,797 DEBUG [M:0;8394b182524b:38791 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8394b182524b:38791 2024-11-13T03:32:35,798 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8394b182524b,38791,1731468754835 2024-11-13T03:32:35,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,810 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8394b182524b,38791,1731468754835 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:35,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:35,841 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T03:32:35,842 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8394b182524b,38791,1731468754835 from backup master directory 2024-11-13T03:32:35,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8394b182524b,38791,1731468754835 2024-11-13T03:32:35,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:32:35,851 WARN [master/8394b182524b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:32:35,851 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8394b182524b,38791,1731468754835 2024-11-13T03:32:35,854 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T03:32:35,855 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T03:32:35,908 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/hbase.id] with ID: 7059c573-6fcb-43cb-b1dd-9918377502a6 2024-11-13T03:32:35,909 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/.tmp/hbase.id 2024-11-13T03:32:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:32:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:32:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:32:35,926 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/.tmp/hbase.id]:[hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/hbase.id] 2024-11-13T03:32:35,970 INFO [master/8394b182524b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:32:35,974 INFO [master/8394b182524b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T03:32:35,992 INFO [master/8394b182524b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T03:32:36,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:32:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:32:36,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:32:36,138 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:32:36,139 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T03:32:36,145 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:32:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:32:36,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:32:36,190 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store 2024-11-13T03:32:36,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:32:36,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:32:36,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:32:36,216 INFO [master/8394b182524b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T03:32:36,219 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:36,221 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:32:36,221 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:36,221 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:36,223 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:32:36,223 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:36,223 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:36,225 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731468756221Disabling compacts and flushes for region at 1731468756221Disabling writes for close at 1731468756223 (+2 ms)Writing region close event to WAL at 1731468756223Closed at 1731468756223 2024-11-13T03:32:36,227 WARN [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/.initializing 2024-11-13T03:32:36,227 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/WALs/8394b182524b,38791,1731468754835 2024-11-13T03:32:36,234 INFO [master/8394b182524b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:36,248 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C38791%2C1731468754835, suffix=, logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/WALs/8394b182524b,38791,1731468754835, archiveDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/oldWALs, maxLogs=10 2024-11-13T03:32:36,274 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/WALs/8394b182524b,38791,1731468754835/8394b182524b%2C38791%2C1731468754835.1731468756253, exclude list is [], retry=0 2024-11-13T03:32:36,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:36,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:36,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T03:32:36,326 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-13T03:32:36,381 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/WALs/8394b182524b,38791,1731468754835/8394b182524b%2C38791%2C1731468754835.1731468756253 2024-11-13T03:32:36,382 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:32:36,383 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:36,383 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:36,386 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,386 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T03:32:36,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:36,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:36,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T03:32:36,451 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:36,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:36,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T03:32:36,455 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:36,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:36,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T03:32:36,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:36,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:36,461 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,464 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,465 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,470 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,471 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,474 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:32:36,477 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:32:36,481 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:36,482 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61094350, jitterRate=-0.08962324261665344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:32:36,489 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731468756398Initializing all the Stores at 1731468756400 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468756400Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468756401 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468756401Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468756401Cleaning up temporary data from old regions at 1731468756471 (+70 ms)Region opened successfully at 1731468756489 (+18 ms) 2024-11-13T03:32:36,491 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T03:32:36,521 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d5593b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:32:36,548 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T03:32:36,557 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T03:32:36,557 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T03:32:36,559 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T03:32:36,561 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T03:32:36,565 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T03:32:36,565 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T03:32:36,587 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T03:32:36,595 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T03:32:36,682 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T03:32:36,687 INFO [master/8394b182524b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T03:32:36,689 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T03:32:36,745 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T03:32:36,750 INFO [master/8394b182524b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T03:32:36,755 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T03:32:36,765 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T03:32:36,767 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T03:32:36,776 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T03:32:36,796 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T03:32:36,807 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,821 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8394b182524b,38791,1731468754835, sessionid=0x101324078460000, setting cluster-up flag (Was=false) 2024-11-13T03:32:36,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,881 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T03:32:36,887 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8394b182524b,38791,1731468754835 2024-11-13T03:32:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:36,944 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T03:32:36,948 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8394b182524b,38791,1731468754835 2024-11-13T03:32:36,956 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T03:32:36,989 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(746): ClusterId : 7059c573-6fcb-43cb-b1dd-9918377502a6 2024-11-13T03:32:36,989 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(746): ClusterId : 7059c573-6fcb-43cb-b1dd-9918377502a6 2024-11-13T03:32:36,991 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(746): ClusterId : 7059c573-6fcb-43cb-b1dd-9918377502a6 2024-11-13T03:32:36,991 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:32:36,991 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:32:36,991 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:32:36,999 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:32:36,999 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:32:36,999 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:32:36,999 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:32:36,999 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:32:36,999 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:32:37,019 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:32:37,019 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:32:37,019 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:32:37,019 DEBUG [RS:0;8394b182524b:38391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bd2db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:32:37,019 DEBUG [RS:2;8394b182524b:46881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae01b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:32:37,019 DEBUG [RS:1;8394b182524b:35657 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4ae978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:32:37,031 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T03:32:37,034 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;8394b182524b:46881 2024-11-13T03:32:37,037 INFO [RS:2;8394b182524b:46881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:32:37,037 INFO [RS:2;8394b182524b:46881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:32:37,037 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:32:37,038 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8394b182524b:38391 2024-11-13T03:32:37,038 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8394b182524b:35657 2024-11-13T03:32:37,039 INFO [RS:0;8394b182524b:38391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:32:37,039 INFO [RS:1;8394b182524b:35657 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:32:37,039 INFO [RS:0;8394b182524b:38391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:32:37,039 INFO [RS:1;8394b182524b:35657 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:32:37,039 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:32:37,039 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:32:37,039 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=46881, startcode=1731468755751 2024-11-13T03:32:37,040 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=35657, startcode=1731468755703 2024-11-13T03:32:37,040 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=38391, startcode=1731468755599 2024-11-13T03:32:37,043 INFO [master/8394b182524b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T03:32:37,050 INFO [master/8394b182524b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T03:32:37,051 DEBUG [RS:2;8394b182524b:46881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:32:37,051 DEBUG [RS:0;8394b182524b:38391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:32:37,051 DEBUG [RS:1;8394b182524b:35657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:32:37,055 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8394b182524b,38791,1731468754835 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T03:32:37,063 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:32:37,063 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:32:37,064 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:32:37,064 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:32:37,064 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8394b182524b:0, corePoolSize=10, maxPoolSize=10 2024-11-13T03:32:37,065 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,065 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:32:37,065 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,076 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731468787076 2024-11-13T03:32:37,078 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T03:32:37,079 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T03:32:37,080 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:32:37,081 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T03:32:37,083 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T03:32:37,084 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T03:32:37,085 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T03:32:37,085 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T03:32:37,085 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54777, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:32:37,086 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52743, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:32:37,086 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37111, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:32:37,087 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:37,088 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T03:32:37,093 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T03:32:37,092 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,097 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T03:32:37,098 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T03:32:37,099 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T03:32:37,101 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T03:32:37,101 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T03:32:37,102 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T03:32:37,102 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T03:32:37,126 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468757125,5,FailOnTimeoutGroup] 2024-11-13T03:32:37,127 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468757126,5,FailOnTimeoutGroup] 2024-11-13T03:32:37,127 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,127 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T03:32:37,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:32:37,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:32:37,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:32:37,128 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,129 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,131 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T03:32:37,131 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed 2024-11-13T03:32:37,139 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T03:32:37,139 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T03:32:37,139 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T03:32:37,139 WARN [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T03:32:37,139 WARN [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T03:32:37,139 WARN [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T03:32:37,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:32:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:32:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:32:37,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:37,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:32:37,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:32:37,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:37,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:37,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:32:37,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:32:37,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:37,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:37,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:32:37,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:32:37,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:37,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:37,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:32:37,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:32:37,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:37,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:37,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:32:37,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740 2024-11-13T03:32:37,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740 2024-11-13T03:32:37,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:32:37,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:32:37,172 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:32:37,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:32:37,179 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:37,180 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71259858, jitterRate=0.06185463070869446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:32:37,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731468757147Initializing all the Stores at 1731468757148 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468757148Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468757149 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468757149Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468757149Cleaning up temporary data from old regions at 1731468757171 (+22 ms)Region opened successfully at 1731468757186 (+15 ms) 2024-11-13T03:32:37,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:32:37,186 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:32:37,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:32:37,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:32:37,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:32:37,188 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:32:37,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731468757186Disabling compacts and flushes for region at 1731468757186Disabling writes for close at 1731468757186Writing region close event to WAL at 1731468757188 (+2 ms)Closed at 1731468757188 2024-11-13T03:32:37,193 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:32:37,193 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T03:32:37,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T03:32:37,211 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:32:37,214 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T03:32:37,241 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=38391, startcode=1731468755599 2024-11-13T03:32:37,241 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=46881, startcode=1731468755751 2024-11-13T03:32:37,241 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,38791,1731468754835 with port=35657, startcode=1731468755703 2024-11-13T03:32:37,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,35657,1731468755703 2024-11-13T03:32:37,245 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(517): Registering regionserver=8394b182524b,35657,1731468755703 2024-11-13T03:32:37,251 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,46881,1731468755751 2024-11-13T03:32:37,252 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(517): Registering regionserver=8394b182524b,46881,1731468755751 2024-11-13T03:32:37,252 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed 2024-11-13T03:32:37,252 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33331 2024-11-13T03:32:37,252 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:32:37,255 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,38391,1731468755599 2024-11-13T03:32:37,255 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed 2024-11-13T03:32:37,255 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33331 2024-11-13T03:32:37,255 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:32:37,255 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38791 {}] master.ServerManager(517): Registering regionserver=8394b182524b,38391,1731468755599 2024-11-13T03:32:37,258 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed 2024-11-13T03:32:37,258 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33331 2024-11-13T03:32:37,258 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:32:37,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:32:37,335 DEBUG [RS:1;8394b182524b:35657 {}] zookeeper.ZKUtil(111): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,35657,1731468755703 2024-11-13T03:32:37,336 WARN [RS:1;8394b182524b:35657 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:32:37,336 INFO [RS:1;8394b182524b:35657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:37,336 DEBUG [RS:0;8394b182524b:38391 {}] zookeeper.ZKUtil(111): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,38391,1731468755599 2024-11-13T03:32:37,336 DEBUG [RS:2;8394b182524b:46881 {}] zookeeper.ZKUtil(111): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,46881,1731468755751 2024-11-13T03:32:37,336 WARN [RS:0;8394b182524b:38391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:32:37,336 WARN [RS:2;8394b182524b:46881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:32:37,336 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703 2024-11-13T03:32:37,336 INFO [RS:0;8394b182524b:38391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:37,336 INFO [RS:2;8394b182524b:46881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:37,336 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,38391,1731468755599 2024-11-13T03:32:37,336 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751 2024-11-13T03:32:37,338 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,35657,1731468755703] 2024-11-13T03:32:37,338 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,38391,1731468755599] 2024-11-13T03:32:37,339 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,46881,1731468755751] 2024-11-13T03:32:37,362 INFO [RS:1;8394b182524b:35657 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:32:37,362 INFO [RS:0;8394b182524b:38391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:32:37,362 INFO [RS:2;8394b182524b:46881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:32:37,365 WARN [8394b182524b:38791 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T03:32:37,379 INFO [RS:0;8394b182524b:38391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:32:37,379 INFO [RS:1;8394b182524b:35657 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:32:37,379 INFO [RS:2;8394b182524b:46881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:32:37,384 INFO [RS:0;8394b182524b:38391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:37,384 INFO [RS:2;8394b182524b:46881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:37,384 INFO [RS:1;8394b182524b:35657 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:37,384 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,384 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,384 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,385 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:32:37,385 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:32:37,385 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:32:37,391 INFO [RS:0;8394b182524b:38391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:32:37,391 INFO [RS:2;8394b182524b:46881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:32:37,391 INFO [RS:1;8394b182524b:35657 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:32:37,393 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,393 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,393 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,393 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,393 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,394 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,395 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,395 DEBUG [RS:1;8394b182524b:35657 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,395 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,395 DEBUG [RS:2;8394b182524b:46881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,395 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,396 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,396 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,397 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:32:37,397 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,397 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,397 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,397 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,397 DEBUG [RS:0;8394b182524b:38391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:32:37,397 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,35657,1731468755703-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,398 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,46881,1731468755751-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,400 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38391,1731468755599-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:32:37,421 INFO [RS:1;8394b182524b:35657 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:32:37,421 INFO [RS:2;8394b182524b:46881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:32:37,422 INFO [RS:0;8394b182524b:38391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:32:37,423 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38391,1731468755599-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,423 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,35657,1731468755703-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,423 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,46881,1731468755751-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,423 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,423 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,424 INFO [RS:0;8394b182524b:38391 {}] regionserver.Replication(171): 8394b182524b,38391,1731468755599 started 2024-11-13T03:32:37,424 INFO [RS:1;8394b182524b:35657 {}] regionserver.Replication(171): 8394b182524b,35657,1731468755703 started 2024-11-13T03:32:37,424 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,424 INFO [RS:2;8394b182524b:46881 {}] regionserver.Replication(171): 8394b182524b,46881,1731468755751 started 2024-11-13T03:32:37,441 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,442 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,35657,1731468755703, RpcServer on 8394b182524b/172.17.0.3:35657, sessionid=0x101324078460002 2024-11-13T03:32:37,442 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:32:37,443 DEBUG [RS:1;8394b182524b:35657 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,35657,1731468755703 2024-11-13T03:32:37,443 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,35657,1731468755703' 2024-11-13T03:32:37,443 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:32:37,444 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,35657,1731468755703 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,35657,1731468755703' 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:32:37,445 DEBUG [RS:1;8394b182524b:35657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:32:37,446 DEBUG [RS:1;8394b182524b:35657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:32:37,446 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,446 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:37,446 INFO [RS:1;8394b182524b:35657 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:32:37,446 INFO [RS:1;8394b182524b:35657 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:32:37,446 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,38391,1731468755599, RpcServer on 8394b182524b/172.17.0.3:38391, sessionid=0x101324078460001 2024-11-13T03:32:37,446 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,46881,1731468755751, RpcServer on 8394b182524b/172.17.0.3:46881, sessionid=0x101324078460003 2024-11-13T03:32:37,446 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:32:37,446 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:32:37,446 DEBUG [RS:2;8394b182524b:46881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,46881,1731468755751 2024-11-13T03:32:37,446 DEBUG [RS:0;8394b182524b:38391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,38391,1731468755599 2024-11-13T03:32:37,447 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,46881,1731468755751' 2024-11-13T03:32:37,447 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,38391,1731468755599' 2024-11-13T03:32:37,447 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:32:37,447 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:32:37,447 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:32:37,447 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:32:37,448 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:32:37,448 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:32:37,448 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:32:37,448 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:32:37,448 DEBUG [RS:0;8394b182524b:38391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,38391,1731468755599 2024-11-13T03:32:37,448 DEBUG [RS:2;8394b182524b:46881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,46881,1731468755751 2024-11-13T03:32:37,448 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,38391,1731468755599' 2024-11-13T03:32:37,448 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,46881,1731468755751' 2024-11-13T03:32:37,448 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:32:37,448 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:32:37,449 DEBUG [RS:0;8394b182524b:38391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:32:37,449 DEBUG [RS:2;8394b182524b:46881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:32:37,449 DEBUG [RS:0;8394b182524b:38391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:32:37,449 INFO [RS:0;8394b182524b:38391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:32:37,449 DEBUG [RS:2;8394b182524b:46881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:32:37,449 INFO [RS:0;8394b182524b:38391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:32:37,449 INFO [RS:2;8394b182524b:46881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:32:37,449 INFO [RS:2;8394b182524b:46881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:32:37,557 INFO [RS:1;8394b182524b:35657 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:37,557 INFO [RS:2;8394b182524b:46881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:37,557 INFO [RS:0;8394b182524b:38391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:37,559 INFO [RS:0;8394b182524b:38391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C38391%2C1731468755599, suffix=, logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,38391,1731468755599, archiveDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs, maxLogs=32 2024-11-13T03:32:37,559 INFO [RS:2;8394b182524b:46881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C46881%2C1731468755751, suffix=, logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751, archiveDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs, maxLogs=32 2024-11-13T03:32:37,559 INFO [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C35657%2C1731468755703, suffix=, logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703, archiveDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs, maxLogs=32 2024-11-13T03:32:37,576 DEBUG [RS:0;8394b182524b:38391 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,38391,1731468755599/8394b182524b%2C38391%2C1731468755599.1731468757564, exclude list is [], retry=0 2024-11-13T03:32:37,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:37,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:37,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:37,583 DEBUG [RS:1;8394b182524b:35657 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703/8394b182524b%2C35657%2C1731468755703.1731468757564, exclude list is [], retry=0 2024-11-13T03:32:37,583 DEBUG [RS:2;8394b182524b:46881 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751/8394b182524b%2C46881%2C1731468755751.1731468757564, exclude list is [], retry=0 2024-11-13T03:32:37,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:37,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:37,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:37,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:37,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:37,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:37,613 INFO [RS:0;8394b182524b:38391 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,38391,1731468755599/8394b182524b%2C38391%2C1731468755599.1731468757564 2024-11-13T03:32:37,613 DEBUG [RS:0;8394b182524b:38391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:37,626 INFO [RS:2;8394b182524b:46881 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751/8394b182524b%2C46881%2C1731468755751.1731468757564 2024-11-13T03:32:37,626 DEBUG [RS:2;8394b182524b:46881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:37,628 INFO [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703/8394b182524b%2C35657%2C1731468755703.1731468757564 2024-11-13T03:32:37,628 DEBUG [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:37,869 DEBUG [8394b182524b:38791 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T03:32:37,880 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:37,886 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T03:32:37,887 INFO [8394b182524b:38791 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:37,887 INFO [8394b182524b:38791 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:37,887 INFO [8394b182524b:38791 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T03:32:37,887 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:37,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8394b182524b,35657,1731468755703 2024-11-13T03:32:37,900 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,35657,1731468755703, state=OPENING 2024-11-13T03:32:37,954 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T03:32:37,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:37,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:37,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:37,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:37,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:37,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:37,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:37,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:37,968 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:32:37,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,35657,1731468755703}] 2024-11-13T03:32:38,150 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:38,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:38,164 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T03:32:38,165 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:38,166 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T03:32:38,169 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C35657%2C1731468755703.meta, suffix=.meta, logDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703, archiveDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs, maxLogs=32 2024-11-13T03:32:38,185 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703/8394b182524b%2C35657%2C1731468755703.meta.1731468758171.meta, exclude list is [], retry=0 2024-11-13T03:32:38,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:38,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:38,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:38,224 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703/8394b182524b%2C35657%2C1731468755703.meta.1731468758171.meta 2024-11-13T03:32:38,225 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:38,225 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:38,226 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T03:32:38,229 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T03:32:38,233 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T03:32:38,237 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T03:32:38,238 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:38,238 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T03:32:38,238 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T03:32:38,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:32:38,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:32:38,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:38,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:38,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:32:38,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:32:38,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:38,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:38,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:32:38,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:32:38,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:38,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:38,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:32:38,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:32:38,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:38,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:38,252 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:32:38,254 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740 2024-11-13T03:32:38,256 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740 2024-11-13T03:32:38,258 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:32:38,258 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:32:38,259 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:32:38,261 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:32:38,263 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67754530, jitterRate=0.00962117314338684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:32:38,263 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T03:32:38,264 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731468758238Writing region info on filesystem at 1731468758239 (+1 ms)Initializing all the Stores at 1731468758240 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468758241 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468758241Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468758241Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468758241Cleaning up temporary data from old regions at 1731468758258 (+17 ms)Running coprocessor post-open hooks at 1731468758263 (+5 ms)Region opened successfully at 1731468758264 (+1 ms) 2024-11-13T03:32:38,271 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731468758139 2024-11-13T03:32:38,281 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T03:32:38,281 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T03:32:38,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8394b182524b,35657,1731468755703 2024-11-13T03:32:38,285 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,35657,1731468755703, state=OPEN 2024-11-13T03:32:38,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:38,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:38,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:38,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:38,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:38,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:38,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:38,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:38,336 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8394b182524b,35657,1731468755703 2024-11-13T03:32:38,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T03:32:38,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,35657,1731468755703 in 367 msec 2024-11-13T03:32:38,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T03:32:38,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1450 sec 2024-11-13T03:32:38,353 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:32:38,353 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T03:32:38,370 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:38,372 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,35657,1731468755703, seqNum=-1] 2024-11-13T03:32:38,390 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:38,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:38,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4260 sec 2024-11-13T03:32:38,412 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731468758412, completionTime=-1 2024-11-13T03:32:38,414 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T03:32:38,415 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T03:32:38,444 INFO [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T03:32:38,444 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731468818444 2024-11-13T03:32:38,444 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731468878444 2024-11-13T03:32:38,444 INFO [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-13T03:32:38,446 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T03:32:38,452 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,453 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,453 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,454 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8394b182524b:38791, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,455 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,456 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,462 DEBUG [master/8394b182524b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T03:32:38,486 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.634sec 2024-11-13T03:32:38,488 INFO [master/8394b182524b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T03:32:38,489 INFO [master/8394b182524b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T03:32:38,490 INFO [master/8394b182524b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T03:32:38,490 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T03:32:38,490 INFO [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T03:32:38,491 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:32:38,492 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T03:32:38,496 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T03:32:38,496 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T03:32:38,497 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,38791,1731468754835-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:38,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7bb128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:32:38,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8394b182524b,38791,-1 for getting cluster id 2024-11-13T03:32:38,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:32:38,513 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7059c573-6fcb-43cb-b1dd-9918377502a6' 2024-11-13T03:32:38,515 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:32:38,515 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7059c573-6fcb-43cb-b1dd-9918377502a6" 2024-11-13T03:32:38,515 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4792eea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:32:38,516 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8394b182524b,38791,-1] 2024-11-13T03:32:38,518 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:32:38,521 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:38,521 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:32:38,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6423f813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:32:38,524 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:38,531 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,35657,1731468755703, seqNum=-1] 2024-11-13T03:32:38,532 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:38,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:38,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8394b182524b,38791,1731468754835 2024-11-13T03:32:38,552 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33331/hbase 2024-11-13T03:32:38,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=3223 2024-11-13T03:32:38,587 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:38,590 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:38,592 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:38,596 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58545947, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-58545947, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:38,610 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58545947/hregion-58545947.1731468758597, exclude list is [], retry=0 2024-11-13T03:32:38,614 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:38,614 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:38,614 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:38,645 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58545947/hregion-58545947.1731468758597 2024-11-13T03:32:38,645 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:32:38,646 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9395bcd6c20e1f36a600b03dba715267, NAME => 'testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:38,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:32:38,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:32:38,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:32:38,660 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:38,662 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,664 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9395bcd6c20e1f36a600b03dba715267 columnFamilyName a 2024-11-13T03:32:38,665 DEBUG [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:38,665 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] regionserver.HStore(327): Store=9395bcd6c20e1f36a600b03dba715267/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:38,666 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,667 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,668 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,669 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,669 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,672 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:38,676 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:38,677 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9395bcd6c20e1f36a600b03dba715267; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75185071, jitterRate=0.1203448623418808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:38,678 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9395bcd6c20e1f36a600b03dba715267: Writing region info on filesystem at 1731468758660Initializing all the Stores at 1731468758662 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468758662Cleaning up temporary data from old regions at 1731468758669 (+7 ms)Region opened successfully at 1731468758678 (+9 ms) 2024-11-13T03:32:38,678 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9395bcd6c20e1f36a600b03dba715267, disabling compactions & flushes 2024-11-13T03:32:38,678 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. 2024-11-13T03:32:38,678 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. 2024-11-13T03:32:38,678 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. after waiting 0 ms 2024-11-13T03:32:38,679 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. 2024-11-13T03:32:38,679 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. 2024-11-13T03:32:38,679 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9395bcd6c20e1f36a600b03dba715267: Waiting for close lock at 1731468758678Disabling compacts and flushes for region at 1731468758678Disabling writes for close at 1731468758679 (+1 ms)Writing region close event to WAL at 1731468758679Closed at 1731468758679 2024-11-13T03:32:38,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741837_1013 (size=175) 2024-11-13T03:32:38,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741837_1013 (size=175) 2024-11-13T03:32:38,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741837_1013 (size=175) 2024-11-13T03:32:38,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:38,690 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58545947:(num 1731468758597) 2024-11-13T03:32:38,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:32:38,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741839_1015 (size=469) 2024-11-13T03:32:38,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741839_1015 (size=469) 2024-11-13T03:32:38,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741839_1015 (size=469) 2024-11-13T03:32:38,736 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:32:38,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741840_1016 (size=322) 2024-11-13T03:32:38,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741840_1016 (size=322) 2024-11-13T03:32:38,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741840_1016 (size=322) 2024-11-13T03:32:38,796 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1, size=469 (469bytes) 2024-11-13T03:32:38,797 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T03:32:38,797 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T03:32:38,797 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1 2024-11-13T03:32:38,803 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1 after 3ms 2024-11-13T03:32:38,835 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:38,836 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1 took 41ms 2024-11-13T03:32:38,841 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1 so closing down 2024-11-13T03:32:38,842 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:38,845 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T03:32:38,876 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T03:32:38,878 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741841_1017 (size=469) 2024-11-13T03:32:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741841_1017 (size=469) 2024-11-13T03:32:38,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741841_1017 (size=469) 2024-11-13T03:32:38,892 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:38,895 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002 2024-11-13T03:32:38,900 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T03:32:38,900 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1, journal: Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1, size=469 (469bytes) at 1731468758797Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1 so closing down at 1731468758842 (+45 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp at 1731468758876 (+34 ms)3 split writer threads finished at 1731468758878 (+2 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468758892 (+14 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002 at 1731468758895 (+3 ms)Processed 2 edits across 1 Regions in 59 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731468758900 (+5 ms) 2024-11-13T03:32:38,917 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2, size=322 (322bytes) 2024-11-13T03:32:38,917 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2 2024-11-13T03:32:38,918 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2 after 1ms 2024-11-13T03:32:38,947 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:38,947 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2 took 30ms 2024-11-13T03:32:38,950 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2 so closing down 2024-11-13T03:32:38,950 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:38,953 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T03:32:38,981 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T03:32:38,981 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:38,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741842_1018 (size=322) 2024-11-13T03:32:38,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741842_1018 (size=322) 2024-11-13T03:32:38,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741842_1018 (size=322) 2024-11-13T03:32:38,993 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:39,024 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,027 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-13T03:32:39,029 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T03:32:39,029 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2, journal: Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2, size=322 (322bytes) at 1731468758917Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2 so closing down at 1731468758950 (+33 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002-wal-2.temp at 1731468758981 (+31 ms)3 split writer threads finished at 1731468758981Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468758993 (+12 ms)Processed 1 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731468759029 (+36 ms) 2024-11-13T03:32:39,029 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:39,032 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:39,045 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal.1731468759032, exclude list is [], retry=0 2024-11-13T03:32:39,050 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:39,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:39,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:39,080 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal.1731468759032 2024-11-13T03:32:39,081 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:39,081 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9395bcd6c20e1f36a600b03dba715267, NAME => 'testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:39,081 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:39,081 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,081 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,084 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,086 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9395bcd6c20e1f36a600b03dba715267 columnFamilyName a 2024-11-13T03:32:39,086 DEBUG [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:39,087 INFO [StoreOpener-9395bcd6c20e1f36a600b03dba715267-1 {}] regionserver.HStore(327): Store=9395bcd6c20e1f36a600b03dba715267/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:39,087 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,089 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,094 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002 2024-11-13T03:32:39,124 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,129 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002 2024-11-13T03:32:39,132 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9395bcd6c20e1f36a600b03dba715267 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T03:32:39,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/.tmp/a/49a14a9b54e440c986a1db7b8bfe8946 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731468758691/Put/seqid=0 2024-11-13T03:32:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:32:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:32:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:32:39,236 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/.tmp/a/49a14a9b54e440c986a1db7b8bfe8946 2024-11-13T03:32:39,277 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/.tmp/a/49a14a9b54e440c986a1db7b8bfe8946 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/a/49a14a9b54e440c986a1db7b8bfe8946 2024-11-13T03:32:39,287 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/a/49a14a9b54e440c986a1db7b8bfe8946, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T03:32:39,294 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9395bcd6c20e1f36a600b03dba715267 in 158ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T03:32:39,296 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/0000000000000000002 2024-11-13T03:32:39,297 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,297 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,301 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9395bcd6c20e1f36a600b03dba715267 2024-11-13T03:32:39,304 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/9395bcd6c20e1f36a600b03dba715267/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T03:32:39,306 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9395bcd6c20e1f36a600b03dba715267; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68595317, jitterRate=0.02214987576007843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:39,307 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9395bcd6c20e1f36a600b03dba715267: Writing region info on filesystem at 1731468759082Initializing all the Stores at 1731468759083 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759083Obtaining lock to block concurrent updates at 1731468759132 (+49 ms)Preparing flush snapshotting stores in 9395bcd6c20e1f36a600b03dba715267 at 1731468759132Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731468759135 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731468758588.9395bcd6c20e1f36a600b03dba715267. at 1731468759135Flushing 9395bcd6c20e1f36a600b03dba715267/a: creating writer at 1731468759137 (+2 ms)Flushing 9395bcd6c20e1f36a600b03dba715267/a: appending metadata at 1731468759199 (+62 ms)Flushing 9395bcd6c20e1f36a600b03dba715267/a: closing flushed file at 1731468759201 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c20606c: reopening flushed file at 1731468759275 (+74 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9395bcd6c20e1f36a600b03dba715267 in 158ms, sequenceid=2, compaction requested=false; wal=null at 1731468759294 (+19 ms)Cleaning up temporary data from old regions at 1731468759297 (+3 ms)Region opened successfully at 1731468759307 (+10 ms) 2024-11-13T03:32:39,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=377 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:44144 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:50086 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:57722 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:50246 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33331/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:57710 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=3102 (was 3223) 2024-11-13T03:32:39,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=3101 2024-11-13T03:32:39,356 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:39,359 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:39,360 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:39,363 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49438584, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-49438584, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:39,376 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49438584/hregion-49438584.1731468759364, exclude list is [], retry=0 2024-11-13T03:32:39,381 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:39,381 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:39,381 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:39,409 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49438584/hregion-49438584.1731468759364 2024-11-13T03:32:39,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:32:39,410 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c21aea69f7d3aaaf5849809717126c04, NAME => 'testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:32:39,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:32:39,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:32:39,426 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:39,428 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,430 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c21aea69f7d3aaaf5849809717126c04 columnFamilyName a 2024-11-13T03:32:39,430 DEBUG [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:39,431 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] regionserver.HStore(327): Store=c21aea69f7d3aaaf5849809717126c04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:39,431 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,432 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,433 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,433 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,435 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,438 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:39,439 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c21aea69f7d3aaaf5849809717126c04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70431946, jitterRate=0.04951778054237366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:39,439 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c21aea69f7d3aaaf5849809717126c04: Writing region info on filesystem at 1731468759426Initializing all the Stores at 1731468759427 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759428 (+1 ms)Cleaning up temporary data from old regions at 1731468759433 (+5 ms)Region opened successfully at 1731468759439 (+6 ms) 2024-11-13T03:32:39,439 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c21aea69f7d3aaaf5849809717126c04, disabling compactions & flushes 2024-11-13T03:32:39,439 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. 2024-11-13T03:32:39,439 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. 2024-11-13T03:32:39,439 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. after waiting 0 ms 2024-11-13T03:32:39,439 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. 2024-11-13T03:32:39,440 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. 2024-11-13T03:32:39,440 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c21aea69f7d3aaaf5849809717126c04: Waiting for close lock at 1731468759439Disabling compacts and flushes for region at 1731468759439Disabling writes for close at 1731468759439Writing region close event to WAL at 1731468759440 (+1 ms)Closed at 1731468759440 2024-11-13T03:32:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741845_1021 (size=175) 2024-11-13T03:32:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741845_1021 (size=175) 2024-11-13T03:32:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741845_1021 (size=175) 2024-11-13T03:32:39,447 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:39,447 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49438584:(num 1731468759364) 2024-11-13T03:32:39,448 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:32:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741847_1023 (size=469) 2024-11-13T03:32:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741847_1023 (size=469) 2024-11-13T03:32:39,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741847_1023 (size=469) 2024-11-13T03:32:39,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:32:39,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741848_1024 (size=322) 2024-11-13T03:32:39,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741848_1024 (size=322) 2024-11-13T03:32:39,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741848_1024 (size=322) 2024-11-13T03:32:39,540 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2, size=322 (322bytes) 2024-11-13T03:32:39,540 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2 2024-11-13T03:32:39,541 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2 after 1ms 2024-11-13T03:32:39,569 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,570 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2 took 30ms 2024-11-13T03:32:39,573 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2 so closing down 2024-11-13T03:32:39,573 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:39,575 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T03:32:39,602 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T03:32:39,602 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:39,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741849_1025 (size=322) 2024-11-13T03:32:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741849_1025 (size=322) 2024-11-13T03:32:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741849_1025 (size=322) 2024-11-13T03:32:39,612 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:39,614 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 2024-11-13T03:32:39,614 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T03:32:39,614 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2, journal: Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2, size=322 (322bytes) at 1731468759540Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2 so closing down at 1731468759573 (+33 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp at 1731468759602 (+29 ms)3 split writer threads finished at 1731468759602Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468759612 (+10 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 at 1731468759614 (+2 ms)Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731468759614 2024-11-13T03:32:39,629 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1, size=469 (469bytes) 2024-11-13T03:32:39,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1 2024-11-13T03:32:39,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1 after 0ms 2024-11-13T03:32:39,656 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,656 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1 took 27ms 2024-11-13T03:32:39,658 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1 so closing down 2024-11-13T03:32:39,658 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:39,661 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T03:32:39,685 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T03:32:39,686 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:39,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741850_1026 (size=469) 2024-11-13T03:32:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741850_1026 (size=469) 2024-11-13T03:32:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741850_1026 (size=469) 2024-11-13T03:32:39,693 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:39,726 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,728 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002, length=322 2024-11-13T03:32:39,731 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 2024-11-13T03:32:39,731 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T03:32:39,731 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1, journal: Splitting hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1, size=469 (469bytes) at 1731468759630Finishing writing output for hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1 so closing down at 1731468759658 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp at 1731468759685 (+27 ms)3 split writer threads finished at 1731468759686 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468759693 (+7 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 at 1731468759731 (+38 ms)Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731468759731 2024-11-13T03:32:39,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:39,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:39,747 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal.1731468759734, exclude list is [], retry=0 2024-11-13T03:32:39,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:39,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:39,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:39,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal.1731468759734 2024-11-13T03:32:39,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c21aea69f7d3aaaf5849809717126c04, NAME => 'testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,777 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,779 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,781 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c21aea69f7d3aaaf5849809717126c04 columnFamilyName a 2024-11-13T03:32:39,781 DEBUG [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:39,782 INFO [StoreOpener-c21aea69f7d3aaaf5849809717126c04-1 {}] regionserver.HStore(327): Store=c21aea69f7d3aaaf5849809717126c04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:39,782 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,786 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,787 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 2024-11-13T03:32:39,819 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:39,821 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 2024-11-13T03:32:39,821 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c21aea69f7d3aaaf5849809717126c04 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T03:32:39,837 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/.tmp/a/b512b05e425e4670b9085249b487a490 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731468759447/Put/seqid=0 2024-11-13T03:32:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:32:39,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:32:39,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:32:39,847 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/.tmp/a/b512b05e425e4670b9085249b487a490 2024-11-13T03:32:39,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/.tmp/a/b512b05e425e4670b9085249b487a490 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/a/b512b05e425e4670b9085249b487a490 2024-11-13T03:32:39,865 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/a/b512b05e425e4670b9085249b487a490, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T03:32:39,865 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c21aea69f7d3aaaf5849809717126c04 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T03:32:39,866 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/0000000000000000002 2024-11-13T03:32:39,866 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,867 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,869 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c21aea69f7d3aaaf5849809717126c04 2024-11-13T03:32:39,873 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/c21aea69f7d3aaaf5849809717126c04/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T03:32:39,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c21aea69f7d3aaaf5849809717126c04; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65082147, jitterRate=-0.03020043671131134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:39,875 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c21aea69f7d3aaaf5849809717126c04: Writing region info on filesystem at 1731468759777Initializing all the Stores at 1731468759779 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759779Obtaining lock to block concurrent updates at 1731468759821 (+42 ms)Preparing flush snapshotting stores in c21aea69f7d3aaaf5849809717126c04 at 1731468759821Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731468759821Flushing stores of testReplayEditsWrittenIntoWAL,,1731468759357.c21aea69f7d3aaaf5849809717126c04. at 1731468759821Flushing c21aea69f7d3aaaf5849809717126c04/a: creating writer at 1731468759822 (+1 ms)Flushing c21aea69f7d3aaaf5849809717126c04/a: appending metadata at 1731468759836 (+14 ms)Flushing c21aea69f7d3aaaf5849809717126c04/a: closing flushed file at 1731468759836Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dd075cf: reopening flushed file at 1731468759854 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c21aea69f7d3aaaf5849809717126c04 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1731468759865 (+11 ms)Cleaning up temporary data from old regions at 1731468759867 (+2 ms)Region opened successfully at 1731468759875 (+8 ms) 2024-11-13T03:32:39,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:44240 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:57722 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:44016 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:50086 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:50322 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:57794 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=3094 (was 3101) 2024-11-13T03:32:39,907 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=3094 2024-11-13T03:32:39,927 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:39,929 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:39,931 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:39,934 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79461454, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-79461454, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:39,948 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79461454/hregion-79461454.1731468759935, exclude list is [], retry=0 2024-11-13T03:32:39,952 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:39,952 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:39,952 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:39,978 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79461454/hregion-79461454.1731468759935 2024-11-13T03:32:39,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:39,979 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 84062c1c91b692c0863139064033bbe3, NAME => 'testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:32:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:32:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:32:39,990 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:39,993 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:39,995 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName a 2024-11-13T03:32:39,995 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:39,996 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:39,996 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:39,997 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName b 2024-11-13T03:32:39,997 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:39,998 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:39,998 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,000 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName c 2024-11-13T03:32:40,000 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:40,001 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:40,001 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,003 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,004 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,004 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,005 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:40,006 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:40,009 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:40,010 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 84062c1c91b692c0863139064033bbe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60538660, jitterRate=-0.09790366888046265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:40,010 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 84062c1c91b692c0863139064033bbe3: Writing region info on filesystem at 1731468759991Initializing all the Stores at 1731468759992 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759993 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759993Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468759993Cleaning up temporary data from old regions at 1731468760004 (+11 ms)Region opened successfully at 1731468760010 (+6 ms) 2024-11-13T03:32:40,010 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 84062c1c91b692c0863139064033bbe3, disabling compactions & flushes 2024-11-13T03:32:40,010 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:40,010 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:40,010 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. after waiting 0 ms 2024-11-13T03:32:40,010 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:40,011 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:40,011 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 84062c1c91b692c0863139064033bbe3: Waiting for close lock at 1731468760010Disabling compacts and flushes for region at 1731468760010Disabling writes for close at 1731468760010Writing region close event to WAL at 1731468760011 (+1 ms)Closed at 1731468760011 2024-11-13T03:32:40,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741853_1029 (size=175) 2024-11-13T03:32:40,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741853_1029 (size=175) 2024-11-13T03:32:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741853_1029 (size=175) 2024-11-13T03:32:40,019 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:40,020 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79461454:(num 1731468759935) 2024-11-13T03:32:40,020 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:40,022 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:40,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, exclude list is [], retry=0 2024-11-13T03:32:40,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:40,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:40,043 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:40,071 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 2024-11-13T03:32:40,071 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:32:40,305 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, size=0 (0bytes) 2024-11-13T03:32:40,305 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 might be still open, length is 0 2024-11-13T03:32:40,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 2024-11-13T03:32:40,306 WARN [IPC Server handler 4 on default port 33331 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T03:32:40,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 after 2ms 2024-11-13T03:32:42,951 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:44266 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41693:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44266 dst: /127.0.0.1:41693 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41693 remote=/127.0.0.1:44266]. Total timeout mills is 60000, 57323 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:42,952 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:57824 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57824 dst: /127.0.0.1:39941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:42,952 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:50366 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36333:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50366 dst: /127.0.0.1:36333 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:42,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741855_1032 (size=497775) 2024-11-13T03:32:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741855_1032 (size=497775) 2024-11-13T03:32:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741855_1032 (size=497775) 2024-11-13T03:32:43,757 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:32:43,808 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T03:32:44,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 after 4003ms 2024-11-13T03:32:44,341 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:44,342 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 took 4037ms 2024-11-13T03:32:44,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731468760023.temp 2024-11-13T03:32:44,376 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp 2024-11-13T03:32:44,431 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023; continuing. 2024-11-13T03:32:44,431 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 so closing down 2024-11-13T03:32:44,431 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:44,431 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:44,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741856_1033 (size=497783) 2024-11-13T03:32:44,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741856_1033 (size=497783) 2024-11-13T03:32:44,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741856_1033 (size=497783) 2024-11-13T03:32:44,436 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp (wrote 3002 edits, skipped 0 edits in 46 ms) 2024-11-13T03:32:44,438 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 2024-11-13T03:32:44,438 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:32:44,438 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, journal: Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, size=0 (0bytes) at 1731468760305Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp at 1731468764377 (+4072 ms)Split 1024 edits, skipped 0 edits. at 1731468764394 (+17 ms)Split 2048 edits, skipped 0 edits. at 1731468764414 (+20 ms)Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 so closing down at 1731468764431 (+17 ms)3 split writer threads finished at 1731468764431Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp (wrote 3002 edits, skipped 0 edits in 46 ms) at 1731468764436 (+5 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000000001-wal.1731468760023.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 at 1731468764438 (+2 ms)Processed 3002 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023, size=0, length=0, corrupted=false, cancelled=false at 1731468764438 2024-11-13T03:32:44,440 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468760023 2024-11-13T03:32:44,442 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 2024-11-13T03:32:44,442 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:44,444 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:44,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468764445, exclude list is [], retry=0 2024-11-13T03:32:44,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:44,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:44,462 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:44,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468764445 2024-11-13T03:32:44,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:44,491 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:44,492 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,494 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName a 2024-11-13T03:32:44,494 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:44,494 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:44,494 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,495 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName b 2024-11-13T03:32:44,495 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:44,496 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:44,496 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,497 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84062c1c91b692c0863139064033bbe3 columnFamilyName c 2024-11-13T03:32:44,497 DEBUG [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:44,498 INFO [StoreOpener-84062c1c91b692c0863139064033bbe3-1 {}] regionserver.HStore(327): Store=84062c1c91b692c0863139064033bbe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:44,498 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:44,502 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 2024-11-13T03:32:44,537 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:44,574 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:32:44,903 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 84062c1c91b692c0863139064033bbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:32:44,943 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/6d7ced150b5344cf9f5248bc8b48f1b7 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731468760079/Put/seqid=0 2024-11-13T03:32:44,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:32:44,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:32:44,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:32:44,952 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/6d7ced150b5344cf9f5248bc8b48f1b7 2024-11-13T03:32:44,960 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/6d7ced150b5344cf9f5248bc8b48f1b7 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/a/6d7ced150b5344cf9f5248bc8b48f1b7 2024-11-13T03:32:44,967 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/a/6d7ced150b5344cf9f5248bc8b48f1b7, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T03:32:44,968 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 84062c1c91b692c0863139064033bbe3 in 65ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T03:32:44,993 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:32:44,993 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 84062c1c91b692c0863139064033bbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:32:45,003 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/273f4551139a431da9ed66cd37216f81 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731468760136/Put/seqid=0 2024-11-13T03:32:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:32:45,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:32:45,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:32:45,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/273f4551139a431da9ed66cd37216f81 2024-11-13T03:32:45,039 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/b69980bfe16d4f92b9a582395f97b990 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731468760165/Put/seqid=0 2024-11-13T03:32:45,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:32:45,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:32:45,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:32:45,048 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/b69980bfe16d4f92b9a582395f97b990 2024-11-13T03:32:45,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/a/273f4551139a431da9ed66cd37216f81 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/a/273f4551139a431da9ed66cd37216f81 2024-11-13T03:32:45,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/a/273f4551139a431da9ed66cd37216f81, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T03:32:45,066 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/b69980bfe16d4f92b9a582395f97b990 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/b/b69980bfe16d4f92b9a582395f97b990 2024-11-13T03:32:45,073 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/b/b69980bfe16d4f92b9a582395f97b990, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T03:32:45,073 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 84062c1c91b692c0863139064033bbe3 in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T03:32:45,089 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:32:45,089 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 84062c1c91b692c0863139064033bbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:32:45,099 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/cbeb4fb4eb04469db7351f6269652483 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731468760193/Put/seqid=0 2024-11-13T03:32:45,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:32:45,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:32:45,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:32:45,110 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/cbeb4fb4eb04469db7351f6269652483 2024-11-13T03:32:45,136 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/d5c253feb7bd41ac854bf32cfe629cbe is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731468760233/Put/seqid=0 2024-11-13T03:32:45,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:32:45,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:32:45,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:32:45,146 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/d5c253feb7bd41ac854bf32cfe629cbe 2024-11-13T03:32:45,156 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/b/cbeb4fb4eb04469db7351f6269652483 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/b/cbeb4fb4eb04469db7351f6269652483 2024-11-13T03:32:45,163 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/b/cbeb4fb4eb04469db7351f6269652483, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T03:32:45,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/d5c253feb7bd41ac854bf32cfe629cbe as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/c/d5c253feb7bd41ac854bf32cfe629cbe 2024-11-13T03:32:45,171 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/c/d5c253feb7bd41ac854bf32cfe629cbe, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T03:32:45,171 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 84062c1c91b692c0863139064033bbe3 in 82ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T03:32:45,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T03:32:45,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:45,176 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:32:45,176 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T03:32:45,177 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T03:32:45,177 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:45,178 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T03:32:45,178 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:45,180 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731468760272/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:45,183 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 2024-11-13T03:32:45,184 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:32:45,184 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 84062c1c91b692c0863139064033bbe3 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T03:32:45,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/b60eaa664fa649ae85d97650ed57dc7b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731468760240/Put/seqid=0 2024-11-13T03:32:45,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:32:45,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:32:45,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:32:45,206 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/b60eaa664fa649ae85d97650ed57dc7b 2024-11-13T03:32:45,213 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b60eaa664fa649ae85d97650ed57dc7b 2024-11-13T03:32:45,215 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/.tmp/c/b60eaa664fa649ae85d97650ed57dc7b as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/c/b60eaa664fa649ae85d97650ed57dc7b 2024-11-13T03:32:45,222 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b60eaa664fa649ae85d97650ed57dc7b 2024-11-13T03:32:45,222 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/c/b60eaa664fa649ae85d97650ed57dc7b, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T03:32:45,222 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 84062c1c91b692c0863139064033bbe3 in 38ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T03:32:45,224 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/0000000000000003002 2024-11-13T03:32:45,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:45,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:45,226 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:32:45,228 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 84062c1c91b692c0863139064033bbe3 2024-11-13T03:32:45,231 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenIntoWAL/84062c1c91b692c0863139064033bbe3/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T03:32:45,233 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 84062c1c91b692c0863139064033bbe3; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59046899, jitterRate=-0.12013264000415802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:32:45,233 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 84062c1c91b692c0863139064033bbe3: Writing region info on filesystem at 1731468764491Initializing all the Stores at 1731468764492 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468764492Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468764492Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468764492Cleaning up temporary data from old regions at 1731468765225 (+733 ms)Region opened successfully at 1731468765233 (+8 ms) 2024-11-13T03:32:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 84062c1c91b692c0863139064033bbe3, disabling compactions & flushes 2024-11-13T03:32:45,309 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. after waiting 0 ms 2024-11-13T03:32:45,309 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:45,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468759928.84062c1c91b692c0863139064033bbe3. 2024-11-13T03:32:45,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 84062c1c91b692c0863139064033bbe3: Waiting for close lock at 1731468765309Disabling compacts and flushes for region at 1731468765309Disabling writes for close at 1731468765309Writing region close event to WAL at 1731468765311 (+2 ms)Closed at 1731468765311 2024-11-13T03:32:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741857_1034 (size=175) 2024-11-13T03:32:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741857_1034 (size=175) 2024-11-13T03:32:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741857_1034 (size=175) 2024-11-13T03:32:45,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:45,319 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468764445) 2024-11-13T03:32:45,334 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 387) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33081 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1938685444_22 at /127.0.0.1:42956 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:36277 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33331 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1938685444_22 at /127.0.0.1:42984 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33081 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1938685444_22 at /127.0.0.1:43212 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36277 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1938685444_22 at /127.0.0.1:49710 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33331 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=3031 (was 3094) 2024-11-13T03:32:45,348 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=3031 2024-11-13T03:32:45,370 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:45,373 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:45,374 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:45,377 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71207637, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-71207637, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:45,391 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71207637/hregion-71207637.1731468765378, exclude list is [], retry=0 2024-11-13T03:32:45,394 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:45,395 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:45,395 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:45,427 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71207637/hregion-71207637.1731468765378 2024-11-13T03:32:45,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:45,428 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 044b55dc9c866e32157de0b105c932a8, NAME => 'test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:45,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:32:45,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:32:45,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:32:45,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:45,441 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,443 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName a 2024-11-13T03:32:45,443 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:45,444 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:45,444 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,445 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName b 2024-11-13T03:32:45,446 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:45,446 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:45,446 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,448 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName c 2024-11-13T03:32:45,448 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:45,449 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:45,449 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,450 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,450 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,451 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,451 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,452 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:45,453 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:45,456 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:45,457 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 044b55dc9c866e32157de0b105c932a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66924689, jitterRate=-0.002744421362876892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:45,457 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 044b55dc9c866e32157de0b105c932a8: Writing region info on filesystem at 1731468765440Initializing all the Stores at 1731468765441 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468765441Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468765441Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468765441Cleaning up temporary data from old regions at 1731468765451 (+10 ms)Region opened successfully at 1731468765457 (+6 ms) 2024-11-13T03:32:45,457 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 044b55dc9c866e32157de0b105c932a8, disabling compactions & flushes 2024-11-13T03:32:45,457 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:45,458 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:45,458 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. after waiting 0 ms 2024-11-13T03:32:45,458 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:45,459 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:45,459 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 044b55dc9c866e32157de0b105c932a8: Waiting for close lock at 1731468765457Disabling compacts and flushes for region at 1731468765457Disabling writes for close at 1731468765458 (+1 ms)Writing region close event to WAL at 1731468765459 (+1 ms)Closed at 1731468765459 2024-11-13T03:32:45,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741864_1041 (size=175) 2024-11-13T03:32:45,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741864_1041 (size=175) 2024-11-13T03:32:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741864_1041 (size=175) 2024-11-13T03:32:45,466 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:45,466 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71207637:(num 1731468765378) 2024-11-13T03:32:45,466 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:45,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:45,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, exclude list is [], retry=0 2024-11-13T03:32:45,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:45,486 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:45,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:45,522 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 2024-11-13T03:32:45,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:32:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741866_1043 (size=371388) 2024-11-13T03:32:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741866_1043 (size=371388) 2024-11-13T03:32:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741866_1043 (size=371388) 2024-11-13T03:32:45,676 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, size=362.7 K (371388bytes) 2024-11-13T03:32:45,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 2024-11-13T03:32:45,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 after 0ms 2024-11-13T03:32:45,717 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:45,719 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 took 44ms 2024-11-13T03:32:45,723 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731468765469.temp 2024-11-13T03:32:45,758 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp 2024-11-13T03:32:45,761 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 so closing down 2024-11-13T03:32:45,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:45,771 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:45,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741867_1044 (size=371388) 2024-11-13T03:32:45,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741867_1044 (size=371388) 2024-11-13T03:32:45,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741867_1044 (size=371388) 2024-11-13T03:32:45,777 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-13T03:32:45,778 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp to hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 2024-11-13T03:32:45,779 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-13T03:32:45,779 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, journal: Splitting hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, size=362.7 K (371388bytes) at 1731468765676Split 1024 edits, skipped 0 edits. at 1731468765736 (+60 ms)Split 2048 edits, skipped 0 edits. at 1731468765748 (+12 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp at 1731468765758 (+10 ms)Finishing writing output for hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 so closing down at 1731468765761 (+3 ms)3 split writer threads finished at 1731468765772 (+11 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1731468765777 (+5 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000000001-wal.1731468765469.temp to hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 at 1731468765778 (+1 ms)Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731468765779 (+1 ms) 2024-11-13T03:32:45,781 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765469 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468765469 2024-11-13T03:32:45,782 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 2024-11-13T03:32:45,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:45,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:45,804 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, exclude list is [], retry=0 2024-11-13T03:32:45,807 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:45,807 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:45,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:45,834 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 2024-11-13T03:32:45,835 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:45,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741868_1045 (size=371515) 2024-11-13T03:32:45,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741868_1045 (size=371515) 2024-11-13T03:32:45,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741868_1045 (size=371515) 2024-11-13T03:32:46,005 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, size=362.8 K (371515bytes) 2024-11-13T03:32:46,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 2024-11-13T03:32:46,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 after 1ms 2024-11-13T03:32:46,035 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:46,037 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 took 32ms 2024-11-13T03:32:46,041 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731468765785.temp 2024-11-13T03:32:46,082 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp 2024-11-13T03:32:46,083 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 so closing down 2024-11-13T03:32:46,083 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:46,094 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:46,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741869_1046 (size=371515) 2024-11-13T03:32:46,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741869_1046 (size=371515) 2024-11-13T03:32:46,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741869_1046 (size=371515) 2024-11-13T03:32:46,099 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-13T03:32:46,101 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp to hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 2024-11-13T03:32:46,101 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-13T03:32:46,101 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, journal: Splitting hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, size=362.8 K (371515bytes) at 1731468766005Split 1024 edits, skipped 0 edits. at 1731468766054 (+49 ms)Split 2048 edits, skipped 0 edits. at 1731468766069 (+15 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp at 1731468766082 (+13 ms)Finishing writing output for hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 so closing down at 1731468766083 (+1 ms)3 split writer threads finished at 1731468766094 (+11 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731468766099 (+5 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003001-wal.1731468765785.temp to hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 at 1731468766101 (+2 ms)Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731468766101 2024-11-13T03:32:46,103 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468765785 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468765785 2024-11-13T03:32:46,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 2024-11-13T03:32:46,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:46,107 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/test2727-manual,16010,1731468765369, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:46,122 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468766108, exclude list is [], retry=0 2024-11-13T03:32:46,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:46,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:46,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:46,167 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468765369/wal.1731468766108 2024-11-13T03:32:46,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:46,168 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 044b55dc9c866e32157de0b105c932a8, NAME => 'test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:46,168 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:46,168 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,168 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,170 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,171 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName a 2024-11-13T03:32:46,171 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,172 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,172 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,173 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName b 2024-11-13T03:32:46,173 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,174 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,174 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,175 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 044b55dc9c866e32157de0b105c932a8 columnFamilyName c 2024-11-13T03:32:46,175 DEBUG [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,176 INFO [StoreOpener-044b55dc9c866e32157de0b105c932a8-1 {}] regionserver.HStore(327): Store=044b55dc9c866e32157de0b105c932a8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,176 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,177 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,179 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,180 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 2024-11-13T03:32:46,210 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:46,243 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 2024-11-13T03:32:46,245 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 2024-11-13T03:32:46,284 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:46,336 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 2024-11-13T03:32:46,336 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 044b55dc9c866e32157de0b105c932a8 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T03:32:46,355 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/a/e09e9ec5c5a04e478f7b09d54945d0ef is 41, key is test2727/a:100/1731468765839/Put/seqid=0 2024-11-13T03:32:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:32:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:32:46,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:32:46,362 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/a/e09e9ec5c5a04e478f7b09d54945d0ef 2024-11-13T03:32:46,401 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/b/0e3cfb85c31b48cd9ec141b450b46b95 is 41, key is test2727/b:100/1731468765890/Put/seqid=0 2024-11-13T03:32:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:32:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:32:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:32:46,406 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/b/0e3cfb85c31b48cd9ec141b450b46b95 2024-11-13T03:32:46,438 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/c/f70c956d75024b04b8c26a41f821a46f is 41, key is test2727/c:100/1731468765936/Put/seqid=0 2024-11-13T03:32:46,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:32:46,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:32:46,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:32:46,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/c/f70c956d75024b04b8c26a41f821a46f 2024-11-13T03:32:46,451 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/a/e09e9ec5c5a04e478f7b09d54945d0ef as hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/a/e09e9ec5c5a04e478f7b09d54945d0ef 2024-11-13T03:32:46,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/a/e09e9ec5c5a04e478f7b09d54945d0ef, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T03:32:46,459 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/b/0e3cfb85c31b48cd9ec141b450b46b95 as hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/b/0e3cfb85c31b48cd9ec141b450b46b95 2024-11-13T03:32:46,466 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/b/0e3cfb85c31b48cd9ec141b450b46b95, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T03:32:46,467 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/.tmp/c/f70c956d75024b04b8c26a41f821a46f as hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/c/f70c956d75024b04b8c26a41f821a46f 2024-11-13T03:32:46,475 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/c/f70c956d75024b04b8c26a41f821a46f, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T03:32:46,475 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 044b55dc9c866e32157de0b105c932a8 in 139ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T03:32:46,476 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000003000 2024-11-13T03:32:46,477 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/0000000000000006000 2024-11-13T03:32:46,478 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,478 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,479 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:46,481 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 044b55dc9c866e32157de0b105c932a8 2024-11-13T03:32:46,484 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/test2727/044b55dc9c866e32157de0b105c932a8/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T03:32:46,485 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 044b55dc9c866e32157de0b105c932a8; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68328051, jitterRate=0.018167302012443542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:46,486 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 044b55dc9c866e32157de0b105c932a8: Writing region info on filesystem at 1731468766168Initializing all the Stores at 1731468766169 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766169Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766170 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766170Obtaining lock to block concurrent updates at 1731468766336 (+166 ms)Preparing flush snapshotting stores in 044b55dc9c866e32157de0b105c932a8 at 1731468766336Finished memstore snapshotting test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731468766336Flushing stores of test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. at 1731468766336Flushing 044b55dc9c866e32157de0b105c932a8/a: creating writer at 1731468766337 (+1 ms)Flushing 044b55dc9c866e32157de0b105c932a8/a: appending metadata at 1731468766354 (+17 ms)Flushing 044b55dc9c866e32157de0b105c932a8/a: closing flushed file at 1731468766354Flushing 044b55dc9c866e32157de0b105c932a8/b: creating writer at 1731468766370 (+16 ms)Flushing 044b55dc9c866e32157de0b105c932a8/b: appending metadata at 1731468766398 (+28 ms)Flushing 044b55dc9c866e32157de0b105c932a8/b: closing flushed file at 1731468766398Flushing 044b55dc9c866e32157de0b105c932a8/c: creating writer at 1731468766414 (+16 ms)Flushing 044b55dc9c866e32157de0b105c932a8/c: appending metadata at 1731468766437 (+23 ms)Flushing 044b55dc9c866e32157de0b105c932a8/c: closing flushed file at 1731468766437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7085380a: reopening flushed file at 1731468766450 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31b1f768: reopening flushed file at 1731468766458 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ebf03e5: reopening flushed file at 1731468766466 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 044b55dc9c866e32157de0b105c932a8 in 139ms, sequenceid=6000, compaction requested=false; wal=null at 1731468766475 (+9 ms)Cleaning up temporary data from old regions at 1731468766478 (+3 ms)Region opened successfully at 1731468766486 (+8 ms) 2024-11-13T03:32:46,488 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T03:32:46,488 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 044b55dc9c866e32157de0b105c932a8, disabling compactions & flushes 2024-11-13T03:32:46,488 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:46,488 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:46,488 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. after waiting 0 ms 2024-11-13T03:32:46,488 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:46,490 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731468765371.044b55dc9c866e32157de0b105c932a8. 2024-11-13T03:32:46,490 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 044b55dc9c866e32157de0b105c932a8: Waiting for close lock at 1731468766488Disabling compacts and flushes for region at 1731468766488Disabling writes for close at 1731468766488Writing region close event to WAL at 1731468766489 (+1 ms)Closed at 1731468766489 2024-11-13T03:32:46,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741870_1047 (size=175) 2024-11-13T03:32:46,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741870_1047 (size=175) 2024-11-13T03:32:46,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741870_1047 (size=175) 2024-11-13T03:32:46,496 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:46,496 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468766108) 2024-11-13T03:32:46,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=406 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:49862 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:42956 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:42984 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:43326 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=2824 (was 3031) 2024-11-13T03:32:46,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=2823 2024-11-13T03:32:46,538 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:46,545 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:46,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731468766546 2024-11-13T03:32:46,582 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 2024-11-13T03:32:46,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:46,585 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 853544876354a9b8c9c48a37b0d2d68a, NAME => 'testSequentialEditLogSeqNum,,1731468766539.853544876354a9b8c9c48a37b0d2d68a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:46,585 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731468766539.853544876354a9b8c9c48a37b0d2d68a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:46,585 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,585 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,586 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a doesn't exist for region: 853544876354a9b8c9c48a37b0d2d68a on table testSequentialEditLogSeqNum 2024-11-13T03:32:46,587 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 853544876354a9b8c9c48a37b0d2d68a on table testSequentialEditLogSeqNum 2024-11-13T03:32:46,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:32:46,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:32:46,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:32:46,601 INFO [StoreOpener-853544876354a9b8c9c48a37b0d2d68a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,603 INFO [StoreOpener-853544876354a9b8c9c48a37b0d2d68a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 853544876354a9b8c9c48a37b0d2d68a columnFamilyName a 2024-11-13T03:32:46,603 DEBUG [StoreOpener-853544876354a9b8c9c48a37b0d2d68a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,604 INFO [StoreOpener-853544876354a9b8c9c48a37b0d2d68a-1 {}] regionserver.HStore(327): Store=853544876354a9b8c9c48a37b0d2d68a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,604 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,605 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,607 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 853544876354a9b8c9c48a37b0d2d68a 2024-11-13T03:32:46,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:46,610 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 853544876354a9b8c9c48a37b0d2d68a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69984059, jitterRate=0.04284374415874481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:46,611 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 853544876354a9b8c9c48a37b0d2d68a: Writing region info on filesystem at 1731468766586Initializing all the Stores at 1731468766601 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766601Cleaning up temporary data from old regions at 1731468766605 (+4 ms)Region opened successfully at 1731468766610 (+5 ms) 2024-11-13T03:32:46,621 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 853544876354a9b8c9c48a37b0d2d68a 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T03:32:46,640 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/.tmp/a/a087215d4e8549ffb7ba46f406ddca8c is 81, key is testSequentialEditLogSeqNum/a:x0/1731468766611/Put/seqid=0 2024-11-13T03:32:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:32:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:32:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:32:46,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/.tmp/a/a087215d4e8549ffb7ba46f406ddca8c 2024-11-13T03:32:46,656 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/.tmp/a/a087215d4e8549ffb7ba46f406ddca8c as hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/a/a087215d4e8549ffb7ba46f406ddca8c 2024-11-13T03:32:46,664 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/a/a087215d4e8549ffb7ba46f406ddca8c, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T03:32:46,665 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 853544876354a9b8c9c48a37b0d2d68a in 44ms, sequenceid=13, compaction requested=false 2024-11-13T03:32:46,665 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 853544876354a9b8c9c48a37b0d2d68a: 2024-11-13T03:32:46,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:32:46,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:32:46,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:32:46,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:32:46,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:32:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741874_1051 (size=3256) 2024-11-13T03:32:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741874_1051 (size=3256) 2024-11-13T03:32:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741874_1051 (size=3256) 2024-11-13T03:32:46,689 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546, size=3.2 K (3256bytes) 2024-11-13T03:32:46,690 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 2024-11-13T03:32:46,690 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 after 0ms 2024-11-13T03:32:46,717 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:46,717 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 took 28ms 2024-11-13T03:32:46,720 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 so closing down 2024-11-13T03:32:46,720 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:46,721 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468766546.temp 2024-11-13T03:32:46,748 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp 2024-11-13T03:32:46,749 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741877_1054 (size=2650) 2024-11-13T03:32:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741877_1054 (size=2650) 2024-11-13T03:32:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741877_1054 (size=2650) 2024-11-13T03:32:46,757 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:46,758 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp to hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000020 2024-11-13T03:32:46,758 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-13T03:32:46,759 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546, journal: Splitting hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546, size=3.2 K (3256bytes) at 1731468766689Finishing writing output for hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546 so closing down at 1731468766720 (+31 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp at 1731468766748 (+28 ms)3 split writer threads finished at 1731468766749 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731468766757 (+8 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000003-wal.1731468766546.temp to hdfs://localhost:33331/hbase/data/default/testSequentialEditLogSeqNum/853544876354a9b8c9c48a37b0d2d68a/recovered.edits/0000000000000000020 at 1731468766758 (+1 ms)Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468766537/wal.1731468766546, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731468766758 2024-11-13T03:32:46,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:49862 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:42956 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:42984 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=2823 (was 2823) 2024-11-13T03:32:46,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=2822 2024-11-13T03:32:46,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:46,802 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:46,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:46,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19255973, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-19255973, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:46,860 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19255973/hregion-19255973.1731468766839, exclude list is [], retry=0 2024-11-13T03:32:46,863 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:46,863 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:46,864 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:46,889 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19255973/hregion-19255973.1731468766839 2024-11-13T03:32:46,889 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:46,889 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0a6f9bc7cfe76a44465f7e67c26e61d8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:32:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:32:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:32:46,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:46,900 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,902 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName a 2024-11-13T03:32:46,902 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,902 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,902 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,904 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName b 2024-11-13T03:32:46,904 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,905 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,905 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,907 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName c 2024-11-13T03:32:46,907 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,907 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,907 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,908 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,910 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,910 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,910 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:46,911 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,914 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:46,914 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0a6f9bc7cfe76a44465f7e67c26e61d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73348251, jitterRate=0.09297411143779755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:46,915 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: Writing region info on filesystem at 1731468766899Initializing all the Stores at 1731468766900 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766900Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766900Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766900Cleaning up temporary data from old regions at 1731468766910 (+10 ms)Region opened successfully at 1731468766915 (+5 ms) 2024-11-13T03:32:46,915 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0a6f9bc7cfe76a44465f7e67c26e61d8, disabling compactions & flushes 2024-11-13T03:32:46,915 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:46,915 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:46,915 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. after waiting 0 ms 2024-11-13T03:32:46,915 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:46,916 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:46,916 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: Waiting for close lock at 1731468766915Disabling compacts and flushes for region at 1731468766915Disabling writes for close at 1731468766915Writing region close event to WAL at 1731468766916 (+1 ms)Closed at 1731468766916 2024-11-13T03:32:46,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741878_1055 (size=175) 2024-11-13T03:32:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741878_1055 (size=175) 2024-11-13T03:32:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741878_1055 (size=175) 2024-11-13T03:32:46,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:46,923 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19255973:(num 1731468766839) 2024-11-13T03:32:46,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:46,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:46,940 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, exclude list is [], retry=0 2024-11-13T03:32:46,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:46,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:46,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:46,971 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 2024-11-13T03:32:46,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a6f9bc7cfe76a44465f7e67c26e61d8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,974 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,975 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName a 2024-11-13T03:32:46,975 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,976 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,976 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,976 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName b 2024-11-13T03:32:46,977 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,977 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,977 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,978 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName c 2024-11-13T03:32:46,978 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:46,978 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:46,978 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,980 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,981 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,981 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,982 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:46,983 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:46,983 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0a6f9bc7cfe76a44465f7e67c26e61d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74350703, jitterRate=0.10791181027889252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:46,984 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: Writing region info on filesystem at 1731468766972Initializing all the Stores at 1731468766973 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766973Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766974 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468766974Cleaning up temporary data from old regions at 1731468766981 (+7 ms)Region opened successfully at 1731468766984 (+3 ms) 2024-11-13T03:32:46,988 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731468766987/Put/seqid=0 2024-11-13T03:32:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:32:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:32:46,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:32:46,996 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33331/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 0a6f9bc7cfe76a44465f7e67c26e61d8/a 2024-11-13T03:32:47,004 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T03:32:47,005 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:32:47,005 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: 2024-11-13T03:32:47,007 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/98ce53dca74243f19515aa3bbbebc654_SeqId_3_ 2024-11-13T03:32:47,008 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33331/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0a6f9bc7cfe76a44465f7e67c26e61d8/a as hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/98ce53dca74243f19515aa3bbbebc654_SeqId_3_ - updating store file list. 2024-11-13T03:32:47,014 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 98ce53dca74243f19515aa3bbbebc654_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:47,015 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/98ce53dca74243f19515aa3bbbebc654_SeqId_3_ into 0a6f9bc7cfe76a44465f7e67c26e61d8/a 2024-11-13T03:32:47,015 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33331/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0a6f9bc7cfe76a44465f7e67c26e61d8/a (new location: hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/98ce53dca74243f19515aa3bbbebc654_SeqId_3_) 2024-11-13T03:32:47,054 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, size=0 (0bytes) 2024-11-13T03:32:47,054 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 might be still open, length is 0 2024-11-13T03:32:47,054 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 2024-11-13T03:32:47,055 WARN [IPC Server handler 0 on default port 33331 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T03:32:47,055 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 after 1ms 2024-11-13T03:32:48,943 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:43174 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36333:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43174 dst: /127.0.0.1:36333 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36333 remote=/127.0.0.1:43174]. Total timeout mills is 60000, 58078 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:48,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:49936 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41693:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49936 dst: /127.0.0.1:41693 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:48,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:43418 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43418 dst: /127.0.0.1:39941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:48,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741880_1059 (size=606) 2024-11-13T03:32:48,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741880_1059 (size=606) 2024-11-13T03:32:51,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 after 4002ms 2024-11-13T03:32:51,094 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:51,094 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 took 4040ms 2024-11-13T03:32:51,096 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926; continuing. 2024-11-13T03:32:51,096 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 so closing down 2024-11-13T03:32:51,096 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:51,098 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731468766926.temp 2024-11-13T03:32:51,124 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp 2024-11-13T03:32:51,125 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:51,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741882_1060 (size=329) 2024-11-13T03:32:51,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741882_1060 (size=329) 2024-11-13T03:32:51,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741882_1060 (size=329) 2024-11-13T03:32:51,135 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:51,137 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp to hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 2024-11-13T03:32:51,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 43 ms; skipped=1; WAL=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:32:51,137 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, journal: Splitting hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, size=0 (0bytes) at 1731468767054Finishing writing output for hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 so closing down at 1731468771096 (+4042 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp at 1731468771124 (+28 ms)3 split writer threads finished at 1731468771125 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468771135 (+10 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005-wal.1731468766926.temp to hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 at 1731468771137 (+2 ms)Processed 2 edits across 1 Regions in 43 ms; skipped=1; WAL=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926, size=0, length=0, corrupted=false, cancelled=false at 1731468771137 2024-11-13T03:32:51,139 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468766926 2024-11-13T03:32:51,140 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 2024-11-13T03:32:51,140 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:51,142 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:51,154 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468771142, exclude list is [], retry=0 2024-11-13T03:32:51,157 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:51,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:51,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:51,184 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468771142 2024-11-13T03:32:51,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:51,185 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a6f9bc7cfe76a44465f7e67c26e61d8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:51,185 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:51,185 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,186 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,187 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,188 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName a 2024-11-13T03:32:51,188 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,195 DEBUG [StoreFileOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 98ce53dca74243f19515aa3bbbebc654_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:51,195 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/98ce53dca74243f19515aa3bbbebc654_SeqId_3_ 2024-11-13T03:32:51,195 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:51,196 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,197 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName b 2024-11-13T03:32:51,197 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,197 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:51,197 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,198 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a6f9bc7cfe76a44465f7e67c26e61d8 columnFamilyName c 2024-11-13T03:32:51,198 DEBUG [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,199 INFO [StoreOpener-0a6f9bc7cfe76a44465f7e67c26e61d8-1 {}] regionserver.HStore(327): Store=0a6f9bc7cfe76a44465f7e67c26e61d8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:51,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,202 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,202 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 2024-11-13T03:32:51,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:51,230 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 2024-11-13T03:32:51,230 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0a6f9bc7cfe76a44465f7e67c26e61d8 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T03:32:51,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/.tmp/a/b47aa0d3c6d84ce8b5c026968cda04e8 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731468767020/Put/seqid=0 2024-11-13T03:32:51,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:51,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:51,253 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/.tmp/a/b47aa0d3c6d84ce8b5c026968cda04e8 2024-11-13T03:32:51,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:51,259 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/.tmp/a/b47aa0d3c6d84ce8b5c026968cda04e8 as hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/b47aa0d3c6d84ce8b5c026968cda04e8 2024-11-13T03:32:51,266 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/a/b47aa0d3c6d84ce8b5c026968cda04e8, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T03:32:51,266 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0a6f9bc7cfe76a44465f7e67c26e61d8 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T03:32:51,267 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/0000000000000000005 2024-11-13T03:32:51,268 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,268 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,269 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:51,271 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0a6f9bc7cfe76a44465f7e67c26e61d8 2024-11-13T03:32:51,273 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0a6f9bc7cfe76a44465f7e67c26e61d8/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T03:32:51,274 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0a6f9bc7cfe76a44465f7e67c26e61d8; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60587046, jitterRate=-0.09718266129493713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:51,274 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: Writing region info on filesystem at 1731468771186Initializing all the Stores at 1731468771187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468771187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468771187Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468771187Obtaining lock to block concurrent updates at 1731468771230 (+43 ms)Preparing flush snapshotting stores in 0a6f9bc7cfe76a44465f7e67c26e61d8 at 1731468771230Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731468771230Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. at 1731468771230Flushing 0a6f9bc7cfe76a44465f7e67c26e61d8/a: creating writer at 1731468771231 (+1 ms)Flushing 0a6f9bc7cfe76a44465f7e67c26e61d8/a: appending metadata at 1731468771244 (+13 ms)Flushing 0a6f9bc7cfe76a44465f7e67c26e61d8/a: closing flushed file at 1731468771245 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aadbb45: reopening flushed file at 1731468771258 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0a6f9bc7cfe76a44465f7e67c26e61d8 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1731468771266 (+8 ms)Cleaning up temporary data from old regions at 1731468771268 (+2 ms)Region opened successfully at 1731468771274 (+6 ms) 2024-11-13T03:32:51,279 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0a6f9bc7cfe76a44465f7e67c26e61d8, disabling compactions & flushes 2024-11-13T03:32:51,279 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:51,279 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:51,279 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. after waiting 0 ms 2024-11-13T03:32:51,279 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:51,280 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731468766801.0a6f9bc7cfe76a44465f7e67c26e61d8. 2024-11-13T03:32:51,280 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0a6f9bc7cfe76a44465f7e67c26e61d8: Waiting for close lock at 1731468771279Disabling compacts and flushes for region at 1731468771279Disabling writes for close at 1731468771279Writing region close event to WAL at 1731468771280 (+1 ms)Closed at 1731468771280 2024-11-13T03:32:51,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741883_1061 (size=175) 2024-11-13T03:32:51,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741883_1061 (size=175) 2024-11-13T03:32:51,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741883_1061 (size=175) 2024-11-13T03:32:51,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:51,286 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468771142) 2024-11-13T03:32:51,298 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 411) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_176102291_22 at /127.0.0.1:43432 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33331 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_176102291_22 at /127.0.0.1:43220 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33331 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=193 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=2808 (was 2822) 2024-11-13T03:32:51,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=193, ProcessCount=11, AvailableMemoryMB=2808 2024-11-13T03:32:51,324 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:51,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:32:51,331 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8394b182524b,38791,1731468754835 2024-11-13T03:32:51,334 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f3d14a6 2024-11-13T03:32:51,335 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:32:51,337 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:32:51,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:32:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T03:32:51,349 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:32:51,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T03:32:51,351 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:32:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:51,366 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed 2024-11-13T03:32:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:51,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:51,376 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,376 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,376 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468771376Disabling compacts and flushes for region at 1731468771376Disabling writes for close at 1731468771376Writing region close event to WAL at 1731468771376Closed at 1731468771376 2024-11-13T03:32:51,378 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:32:51,382 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731468771378"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731468771378"}]},"ts":"1731468771378"} 2024-11-13T03:32:51,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:32:51,386 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:32:51,389 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731468771387"}]},"ts":"1731468771387"} 2024-11-13T03:32:51,396 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T03:32:51,396 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T03:32:51,398 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:51,398 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:51,398 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T03:32:51,398 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:51,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN}] 2024-11-13T03:32:51,401 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN 2024-11-13T03:32:51,402 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN; state=OFFLINE, location=8394b182524b,46881,1731468755751; forceNewPlan=false, retain=false 2024-11-13T03:32:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:51,555 INFO [8394b182524b:38791 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:51,556 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPENING, regionLocation=8394b182524b,46881,1731468755751 2024-11-13T03:32:51,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN because future has completed 2024-11-13T03:32:51,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751}] 2024-11-13T03:32:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:51,714 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:51,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:51,730 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,730 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:51,731 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,731 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:51,731 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,731 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,734 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,736 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:51,736 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,737 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:51,737 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,739 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:51,739 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:51,740 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:51,740 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,741 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,742 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,743 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,743 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,745 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:51,747 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,750 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:51,751 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened adbb549a8a7c08fdb89452cd0ecd22be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64161592, jitterRate=-0.04391777515411377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:51,751 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:51,752 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for adbb549a8a7c08fdb89452cd0ecd22be: Running coprocessor pre-open hook at 1731468771731Writing region info on filesystem at 1731468771731Initializing all the Stores at 1731468771733 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468771733Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468771733Cleaning up temporary data from old regions at 1731468771743 (+10 ms)Running coprocessor post-open hooks at 1731468771751 (+8 ms)Region opened successfully at 1731468771752 (+1 ms) 2024-11-13T03:32:51,754 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., pid=6, masterSystemTime=1731468771714 2024-11-13T03:32:51,756 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,756 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:51,757 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPEN, openSeqNum=2, regionLocation=8394b182524b,46881,1731468755751 2024-11-13T03:32:51,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 because future has completed 2024-11-13T03:32:51,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T03:32:51,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 in 201 msec 2024-11-13T03:32:51,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T03:32:51,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN in 366 msec 2024-11-13T03:32:51,769 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:32:51,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731468771769"}]},"ts":"1731468771769"} 2024-11-13T03:32:51,771 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T03:32:51,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:32:51,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 430 msec 2024-11-13T03:32:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:51,991 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T03:32:51,991 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T03:32:51,994 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T03:32:52,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T03:32:52,003 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T03:32:52,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T03:32:52,017 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=2] 2024-11-13T03:32:52,019 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:52,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:52,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=adbb549a8a7c08fdb89452cd0ecd22be, source=8394b182524b,46881,1731468755751, destination=8394b182524b,38391,1731468755599, warming up region on 8394b182524b,38391,1731468755599 2024-11-13T03:32:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:52,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=adbb549a8a7c08fdb89452cd0ecd22be, source=8394b182524b,46881,1731468755751, destination=8394b182524b,38391,1731468755599, running balancer 2024-11-13T03:32:52,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59767, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE 2024-11-13T03:32:52,043 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE 2024-11-13T03:32:52,046 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=CLOSING, regionLocation=8394b182524b,46881,1731468755751 2024-11-13T03:32:52,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(7855): Warmup {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:52,049 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE because future has completed 2024-11-13T03:32:52,051 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:52,051 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:32:52,051 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751}] 2024-11-13T03:32:52,051 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,052 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,053 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:52,053 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,054 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:52,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468772054Disabling compacts and flushes for region at 1731468772054Disabling writes for close at 1731468772054Writing region close event to WAL at 1731468772055 (+1 ms)Closed at 1731468772055 2024-11-13T03:32:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T03:32:52,211 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,211 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:32:52,212 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:52,212 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,212 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,212 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:52,212 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,212 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing adbb549a8a7c08fdb89452cd0ecd22be 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T03:32:52,231 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/e462ec365760448a9ac0cc9adcbfacad is 35, key is r1/cf1:q/1731468772021/Put/seqid=0 2024-11-13T03:32:52,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:52,241 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/e462ec365760448a9ac0cc9adcbfacad 2024-11-13T03:32:52,249 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/e462ec365760448a9ac0cc9adcbfacad as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad 2024-11-13T03:32:52,256 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T03:32:52,257 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for adbb549a8a7c08fdb89452cd0ecd22be in 45ms, sequenceid=5, compaction requested=false 2024-11-13T03:32:52,257 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T03:32:52,262 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T03:32:52,264 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,264 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468772212Running coprocessor pre-close hooks at 1731468772212Disabling compacts and flushes for region at 1731468772212Disabling writes for close at 1731468772212Obtaining lock to block concurrent updates at 1731468772212Preparing flush snapshotting stores in adbb549a8a7c08fdb89452cd0ecd22be at 1731468772212Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731468772213 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. at 1731468772213Flushing adbb549a8a7c08fdb89452cd0ecd22be/cf1: creating writer at 1731468772213Flushing adbb549a8a7c08fdb89452cd0ecd22be/cf1: appending metadata at 1731468772231 (+18 ms)Flushing adbb549a8a7c08fdb89452cd0ecd22be/cf1: closing flushed file at 1731468772231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c050846: reopening flushed file at 1731468772248 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for adbb549a8a7c08fdb89452cd0ecd22be in 45ms, sequenceid=5, compaction requested=false at 1731468772257 (+9 ms)Writing region close event to WAL at 1731468772258 (+1 ms)Running coprocessor post-close hooks at 1731468772262 (+4 ms)Closed at 1731468772264 (+2 ms) 2024-11-13T03:32:52,265 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding adbb549a8a7c08fdb89452cd0ecd22be move to 8394b182524b,38391,1731468755599 record at close sequenceid=5 2024-11-13T03:32:52,268 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=CLOSED 2024-11-13T03:32:52,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 because future has completed 2024-11-13T03:32:52,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T03:32:52,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 in 222 msec 2024-11-13T03:32:52,278 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE; state=CLOSED, location=8394b182524b,38391,1731468755599; forceNewPlan=false, retain=false 2024-11-13T03:32:52,428 INFO [8394b182524b:38791 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:52,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPENING, regionLocation=8394b182524b,38391,1731468755599 2024-11-13T03:32:52,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE because future has completed 2024-11-13T03:32:52,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599}] 2024-11-13T03:32:52,591 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,591 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:52,592 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,592 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:52,592 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,592 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,594 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,595 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:52,596 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,604 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad 2024-11-13T03:32:52,604 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,604 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,606 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:52,606 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,607 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,607 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,608 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,610 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,611 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,611 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,612 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:52,614 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,615 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened adbb549a8a7c08fdb89452cd0ecd22be; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69477648, jitterRate=0.03529763221740723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:52,615 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,616 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for adbb549a8a7c08fdb89452cd0ecd22be: Running coprocessor pre-open hook at 1731468772592Writing region info on filesystem at 1731468772592Initializing all the Stores at 1731468772594 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468772594Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468772594Cleaning up temporary data from old regions at 1731468772611 (+17 ms)Running coprocessor post-open hooks at 1731468772615 (+4 ms)Region opened successfully at 1731468772616 (+1 ms) 2024-11-13T03:32:52,617 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., pid=9, masterSystemTime=1731468772586 2024-11-13T03:32:52,620 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,620 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPEN, openSeqNum=9, regionLocation=8394b182524b,38391,1731468755599 2024-11-13T03:32:52,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 because future has completed 2024-11-13T03:32:52,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T03:32:52,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 in 192 msec 2024-11-13T03:32:52,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE in 587 msec 2024-11-13T03:32:52,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:52,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:34030 deadline: 1731468832657, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=38391 startCode=1731468755599. As of locationSeqNum=5. 2024-11-13T03:32:52,663 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=38391 startCode=1731468755599. As of locationSeqNum=5. 2024-11-13T03:32:52,664 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=38391 startCode=1731468755599. As of locationSeqNum=5. 2024-11-13T03:32:52,664 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=38391 startCode=1731468755599. As of locationSeqNum=5. 2024-11-13T03:32:52,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:52,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:52,792 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing adbb549a8a7c08fdb89452cd0ecd22be 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T03:32:52,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/788c13a2fad3447d8174e76592d1e05b is 29, key is r1/cf1:/1731468772782/DeleteFamily/seqid=0 2024-11-13T03:32:52,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:52,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:52,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:52,817 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,823 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf2/47d6b93328884b18b0d650fbcbe457f2 is 29, key is r1/cf2:/1731468772782/DeleteFamily/seqid=0 2024-11-13T03:32:52,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:52,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:52,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:52,845 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf2/47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,851 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,852 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/788c13a2fad3447d8174e76592d1e05b as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,860 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,860 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T03:32:52,861 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf2/47d6b93328884b18b0d650fbcbe457f2 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,868 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,868 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T03:32:52,870 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for adbb549a8a7c08fdb89452cd0ecd22be in 78ms, sequenceid=12, compaction requested=false 2024-11-13T03:32:52,870 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for adbb549a8a7c08fdb89452cd0ecd22be: 2024-11-13T03:32:52,873 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T03:32:52,874 DEBUG [Time-limited test {}] regionserver.HStore(1541): adbb549a8a7c08fdb89452cd0ecd22be/cf1 is initiating major compaction (all files) 2024-11-13T03:32:52,874 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:52,874 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:52,875 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of adbb549a8a7c08fdb89452cd0ecd22be/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,875 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad, hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b] into tmpdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp, totalSize=9.5 K 2024-11-13T03:32:52,877 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e462ec365760448a9ac0cc9adcbfacad, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731468772021 2024-11-13T03:32:52,877 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 788c13a2fad3447d8174e76592d1e05b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T03:32:52,887 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): adbb549a8a7c08fdb89452cd0ecd22be#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:32:52,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:52,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:52,901 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf1/e2099433c47043388c66b358b734ae35 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e2099433c47043388c66b358b734ae35 2024-11-13T03:32:52,914 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in adbb549a8a7c08fdb89452cd0ecd22be/cf1 of adbb549a8a7c08fdb89452cd0ecd22be into e2099433c47043388c66b358b734ae35(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:32:52,914 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for adbb549a8a7c08fdb89452cd0ecd22be: 2024-11-13T03:32:52,915 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T03:32:52,915 DEBUG [Time-limited test {}] regionserver.HStore(1541): adbb549a8a7c08fdb89452cd0ecd22be/cf2 is initiating major compaction (all files) 2024-11-13T03:32:52,915 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:52,915 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:52,915 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of adbb549a8a7c08fdb89452cd0ecd22be/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,915 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2] into tmpdir=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp, totalSize=4.8 K 2024-11-13T03:32:52,916 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 47d6b93328884b18b0d650fbcbe457f2, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T03:32:52,921 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): adbb549a8a7c08fdb89452cd0ecd22be#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:32:52,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:52,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:52,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:52,936 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/.tmp/cf2/65662c48bde4417e90008587c1193b64 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/65662c48bde4417e90008587c1193b64 2024-11-13T03:32:52,944 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in adbb549a8a7c08fdb89452cd0ecd22be/cf2 of adbb549a8a7c08fdb89452cd0ecd22be into 65662c48bde4417e90008587c1193b64(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:32:52,944 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for adbb549a8a7c08fdb89452cd0ecd22be: 2024-11-13T03:32:52,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=adbb549a8a7c08fdb89452cd0ecd22be, source=8394b182524b,38391,1731468755599, destination=8394b182524b,46881,1731468755751, warming up region on 8394b182524b,46881,1731468755751 2024-11-13T03:32:52,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=adbb549a8a7c08fdb89452cd0ecd22be, source=8394b182524b,38391,1731468755599, destination=8394b182524b,46881,1731468755751, running balancer 2024-11-13T03:32:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE 2024-11-13T03:32:52,950 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE 2024-11-13T03:32:52,952 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=CLOSING, regionLocation=8394b182524b,38391,1731468755599 2024-11-13T03:32:52,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(7855): Warmup {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:52,953 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,954 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:52,955 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE because future has completed 2024-11-13T03:32:52,955 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:32:52,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599}] 2024-11-13T03:32:52,964 INFO [StoreFileOpener-adbb549a8a7c08fdb89452cd0ecd22be-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,964 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:52,970 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e2099433c47043388c66b358b734ae35 2024-11-13T03:32:52,977 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad 2024-11-13T03:32:52,977 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,977 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:52,978 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:52,979 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:52,987 INFO [StoreFileOpener-adbb549a8a7c08fdb89452cd0ecd22be-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,987 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:52,993 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/65662c48bde4417e90008587c1193b64 2024-11-13T03:32:52,994 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:52,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468772994Disabling compacts and flushes for region at 1731468772994Disabling writes for close at 1731468772994Writing region close event to WAL at 1731468772995 (+1 ms)Closed at 1731468772995 2024-11-13T03:32:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T03:32:53,110 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,110 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:32:53,110 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:53,111 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,111 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,111 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:53,111 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,111 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad, hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b] to archive 2024-11-13T03:32:53,115 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:32:53,119 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e462ec365760448a9ac0cc9adcbfacad 2024-11-13T03:32:53,121 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/788c13a2fad3447d8174e76592d1e05b 2024-11-13T03:32:53,135 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2] to archive 2024-11-13T03:32:53,136 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:32:53,138 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2 to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/47d6b93328884b18b0d650fbcbe457f2 2024-11-13T03:32:53,146 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T03:32:53,147 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,147 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468773110Running coprocessor pre-close hooks at 1731468773110Disabling compacts and flushes for region at 1731468773110Disabling writes for close at 1731468773111 (+1 ms)Writing region close event to WAL at 1731468773140 (+29 ms)Running coprocessor post-close hooks at 1731468773147 (+7 ms)Closed at 1731468773147 2024-11-13T03:32:53,147 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding adbb549a8a7c08fdb89452cd0ecd22be move to 8394b182524b,46881,1731468755751 record at close sequenceid=12 2024-11-13T03:32:53,150 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,151 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=CLOSED 2024-11-13T03:32:53,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 because future has completed 2024-11-13T03:32:53,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T03:32:53,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 in 201 msec 2024-11-13T03:32:53,161 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE; state=CLOSED, location=8394b182524b,46881,1731468755751; forceNewPlan=false, retain=false 2024-11-13T03:32:53,312 INFO [8394b182524b:38791 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:53,312 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPENING, regionLocation=8394b182524b,46881,1731468755751 2024-11-13T03:32:53,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE because future has completed 2024-11-13T03:32:53,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751}] 2024-11-13T03:32:53,471 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,472 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:53,472 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,472 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:53,472 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,472 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,474 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,475 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:53,475 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:53,482 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e2099433c47043388c66b358b734ae35 2024-11-13T03:32:53,482 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:53,482 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,483 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:53,484 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:53,491 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/65662c48bde4417e90008587c1193b64 2024-11-13T03:32:53,491 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:53,491 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,492 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,493 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,494 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,494 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,495 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:53,497 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,498 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened adbb549a8a7c08fdb89452cd0ecd22be; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67042609, jitterRate=-9.872764348983765E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:53,499 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,499 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for adbb549a8a7c08fdb89452cd0ecd22be: Running coprocessor pre-open hook at 1731468773473Writing region info on filesystem at 1731468773473Initializing all the Stores at 1731468773473Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468773473Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468773474 (+1 ms)Cleaning up temporary data from old regions at 1731468773495 (+21 ms)Running coprocessor post-open hooks at 1731468773499 (+4 ms)Region opened successfully at 1731468773499 2024-11-13T03:32:53,500 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., pid=12, masterSystemTime=1731468773468 2024-11-13T03:32:53,503 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,503 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,503 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPEN, openSeqNum=18, regionLocation=8394b182524b,46881,1731468755751 2024-11-13T03:32:53,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 because future has completed 2024-11-13T03:32:53,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T03:32:53,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,46881,1731468755751 in 191 msec 2024-11-13T03:32:53,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, REOPEN/MOVE in 561 msec 2024-11-13T03:32:53,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:53,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:53,555 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 8394b182524b,46881,1731468755751: testing ***** 2024-11-13T03:32:53,555 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T03:32:53,557 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T03:32:53,558 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T03:32:53,562 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T03:32:53,564 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T03:32:53,574 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 402883184 }, "NonHeapMemoryUsage": { "committed": 172097536, "init": 7667712, "max": -1, "used": 169471800 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56173", "tag.serverName": "8394b182524b,35657,1731468755703", "tag.clusterId": "7059c573-6fcb-43cb-b1dd-9918377502a6", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731468755703, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 1, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T03:32:53,578 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38791 {}] master.MasterRpcServices(700): 8394b182524b,46881,1731468755751 reported a fatal error: ***** ABORTING region server 8394b182524b,46881,1731468755751: testing ***** 2024-11-13T03:32:53,581 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,46881,1731468755751' ***** 2024-11-13T03:32:53,581 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T03:32:53,581 INFO [RS:2;8394b182524b:46881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:32:53,581 INFO [RS:2;8394b182524b:46881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T03:32:53,582 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:32:53,582 INFO [RS:2;8394b182524b:46881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T03:32:53,582 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(3091): Received CLOSE for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,582 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(956): aborting server 8394b182524b,46881,1731468755751 2024-11-13T03:32:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38391 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:41204 deadline: 1731468833582, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=46881 startCode=1731468755751. As of locationSeqNum=12. 2024-11-13T03:32:53,582 INFO [RS:2;8394b182524b:46881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:32:53,583 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:32:53,583 INFO [RS:2;8394b182524b:46881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;8394b182524b:46881. 2024-11-13T03:32:53,583 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,583 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,583 DEBUG [RS:2;8394b182524b:46881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:53,583 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:32:53,583 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=46881 startCode=1731468755751. As of locationSeqNum=12. 2024-11-13T03:32:53,583 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,583 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=46881 startCode=1731468755751. As of locationSeqNum=12. 2024-11-13T03:32:53,583 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=46881 startCode=1731468755751. As of locationSeqNum=12. 2024-11-13T03:32:53,583 DEBUG [RS:2;8394b182524b:46881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:53,584 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:32:53,584 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1325): Online Regions={adbb549a8a7c08fdb89452cd0ecd22be=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.} 2024-11-13T03:32:53,584 DEBUG [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1351): Waiting on adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:53,585 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,585 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468773582Running coprocessor pre-close hooks at 1731468773583 (+1 ms)Disabling compacts and flushes for region at 1731468773583Disabling writes for close at 1731468773583Writing region close event to WAL at 1731468773585 (+2 ms)Running coprocessor post-close hooks at 1731468773585Closed at 1731468773585 2024-11-13T03:32:53,585 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:53,635 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:53,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,46881,1731468755751 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:53,690 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,46881,1731468755751 aborting 2024-11-13T03:32:53,690 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,46881,1731468755751 aborting 2024-11-13T03:32:53,690 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=12 from cache 2024-11-13T03:32:53,784 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,46881,1731468755751; all regions closed. 2024-11-13T03:32:53,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741835_1011 (size=2288) 2024-11-13T03:32:53,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741835_1011 (size=2288) 2024-11-13T03:32:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741835_1011 (size=2288) 2024-11-13T03:32:53,791 DEBUG [RS:2;8394b182524b:46881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:53,791 INFO [RS:2;8394b182524b:46881 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:53,791 INFO [RS:2;8394b182524b:46881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:32:53,792 INFO [RS:2;8394b182524b:46881 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T03:32:53,792 INFO [RS:2;8394b182524b:46881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:32:53,792 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:32:53,792 INFO [RS:2;8394b182524b:46881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:32:53,792 INFO [RS:2;8394b182524b:46881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:32:53,792 INFO [RS:2;8394b182524b:46881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:32:53,793 INFO [RS:2;8394b182524b:46881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46881 2024-11-13T03:32:53,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:32:53,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,46881,1731468755751 2024-11-13T03:32:53,870 INFO [RS:2;8394b182524b:46881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:32:53,881 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,46881,1731468755751] 2024-11-13T03:32:53,891 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,46881,1731468755751 already deleted, retry=false 2024-11-13T03:32:53,891 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 8394b182524b,46881,1731468755751 on 8394b182524b,38791,1731468754835 2024-11-13T03:32:53,903 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 8394b182524b,46881,1731468755751, splitWal=true, meta=false 2024-11-13T03:32:53,904 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18] 2024-11-13T03:32:53,906 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 8394b182524b,46881,1731468755751 (carryingMeta=false) 8394b182524b,46881,1731468755751/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@27d3ba77[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T03:32:53,906 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8394b182524b:46881 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:46881 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:53,906 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18, error=java.net.ConnectException: Call to address=8394b182524b:46881 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:46881 2024-11-13T03:32:53,907 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-13T03:32:53,907 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 from cache 2024-11-13T03:32:53,907 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 8394b182524b:46881 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:46881 2024-11-13T03:32:53,908 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 8394b182524b,46881,1731468755751, splitWal=true, meta=false 2024-11-13T03:32:53,909 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 8394b182524b,46881,1731468755751 had 1 regions 2024-11-13T03:32:53,910 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 8394b182524b,46881,1731468755751, splitWal=true, meta=false, isMeta: false 2024-11-13T03:32:53,912 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting 2024-11-13T03:32:53,913 INFO [PEWorker-1 {}] master.SplitWALManager(105): 8394b182524b,46881,1731468755751 WAL count=1, meta=false 2024-11-13T03:32:53,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 8394b182524b%2C46881%2C1731468755751.1731468757564}] 2024-11-13T03:32:53,920 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=8394b182524b,38391,1731468755599 2024-11-13T03:32:53,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C46881%2C1731468755751.1731468757564, worker=8394b182524b,38391,1731468755599}] 2024-11-13T03:32:53,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:53,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46881-0x101324078460003, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:53,981 INFO [RS:2;8394b182524b:46881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:32:53,982 INFO [RS:2;8394b182524b:46881 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,46881,1731468755751; zookeeper connection closed. 2024-11-13T03:32:53,982 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73f6f89d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73f6f89d 2024-11-13T03:32:54,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38391 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T03:32:54,098 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564, size=2.2 K (2288bytes) 2024-11-13T03:32:54,098 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 2024-11-13T03:32:54,099 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 after 1ms 2024-11-13T03:32:54,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:54,124 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 took 26ms 2024-11-13T03:32:54,130 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for adbb549a8a7c08fdb89452cd0ecd22be: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T03:32:54,130 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 so closing down 2024-11-13T03:32:54,130 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:54,131 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:54,131 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-13T03:32:54,131 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564, journal: Splitting hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564, size=2.2 K (2288bytes) at 1731468774098Finishing writing output for hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 so closing down at 1731468774130 (+32 ms)3 split writer threads finished at 1731468774131 (+1 ms)Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731468774131 2024-11-13T03:32:54,131 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 2024-11-13T03:32:54,132 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T03:32:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38791 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T03:32:54,138 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting/8394b182524b%2C46881%2C1731468755751.1731468757564 to hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs 2024-11-13T03:32:54,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T03:32:54,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C46881%2C1731468755751.1731468757564, worker=8394b182524b,38391,1731468755599 in 216 msec 2024-11-13T03:32:54,142 DEBUG [PEWorker-3 {}] master.SplitWALManager(172): Release split WAL worker=8394b182524b,38391,1731468755599 2024-11-13T03:32:54,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T03:32:54,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 8394b182524b%2C46881%2C1731468755751.1731468757564, worker=8394b182524b,38391,1731468755599 in 229 msec 2024-11-13T03:32:54,146 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting dir is empty, no logs to split. 2024-11-13T03:32:54,146 INFO [PEWorker-1 {}] master.SplitWALManager(105): 8394b182524b,46881,1731468755751 WAL count=0, meta=false 2024-11-13T03:32:54,146 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 8394b182524b,46881,1731468755751 WAL splitting is done? wals=0, meta=false 2024-11-13T03:32:54,148 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 8394b182524b,46881,1731468755751 failed, ignore...File hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,46881,1731468755751-splitting does not exist. 2024-11-13T03:32:54,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN}] 2024-11-13T03:32:54,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN 2024-11-13T03:32:54,153 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T03:32:54,224 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18] 2024-11-13T03:32:54,224 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8394b182524b:46881 this server is in the failed servers list 2024-11-13T03:32:54,225 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8394b182524b:46881 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:46881 2024-11-13T03:32:54,226 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:46881 2024-11-13T03:32:54,226 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,46881,1731468755751, seqNum=18 from cache 2024-11-13T03:32:54,303 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:54,303 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:54,303 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:54,303 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:54,304 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:54,304 INFO [8394b182524b:38791 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:54,304 INFO [8394b182524b:38791 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:54,304 DEBUG [8394b182524b:38791 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:54,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPENING, regionLocation=8394b182524b,38391,1731468755599 2024-11-13T03:32:54,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN because future has completed 2024-11-13T03:32:54,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599}] 2024-11-13T03:32:54,473 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:54,473 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => adbb549a8a7c08fdb89452cd0ecd22be, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:54,474 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,474 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:54,475 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,475 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,477 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,479 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf1 2024-11-13T03:32:54,479 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,486 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf1/e2099433c47043388c66b358b734ae35 2024-11-13T03:32:54,486 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,486 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,487 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adbb549a8a7c08fdb89452cd0ecd22be columnFamilyName cf2 2024-11-13T03:32:54,488 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,494 DEBUG [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/cf2/65662c48bde4417e90008587c1193b64 2024-11-13T03:32:54,494 INFO [StoreOpener-adbb549a8a7c08fdb89452cd0ecd22be-1 {}] regionserver.HStore(327): Store=adbb549a8a7c08fdb89452cd0ecd22be/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,494 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,495 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,496 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,497 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,497 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,498 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:54,499 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,500 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened adbb549a8a7c08fdb89452cd0ecd22be; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67108292, jitterRate=-8.52346420288086E-6}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:54,500 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:32:54,500 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for adbb549a8a7c08fdb89452cd0ecd22be: Running coprocessor pre-open hook at 1731468774475Writing region info on filesystem at 1731468774475Initializing all the Stores at 1731468774476 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774476Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774477 (+1 ms)Cleaning up temporary data from old regions at 1731468774497 (+20 ms)Running coprocessor post-open hooks at 1731468774500 (+3 ms)Region opened successfully at 1731468774500 2024-11-13T03:32:54,502 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., pid=17, masterSystemTime=1731468774461 2024-11-13T03:32:54,504 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:54,504 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:32:54,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=adbb549a8a7c08fdb89452cd0ecd22be, regionState=OPEN, openSeqNum=18, regionLocation=8394b182524b,38391,1731468755599 2024-11-13T03:32:54,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 because future has completed 2024-11-13T03:32:54,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T03:32:54,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure adbb549a8a7c08fdb89452cd0ecd22be, server=8394b182524b,38391,1731468755599 in 199 msec 2024-11-13T03:32:54,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T03:32:54,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=adbb549a8a7c08fdb89452cd0ecd22be, ASSIGN in 359 msec 2024-11-13T03:32:54,512 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 8394b182524b,46881,1731468755751 after splitting done 2024-11-13T03:32:54,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 8394b182524b,46881,1731468755751, splitWal=true, meta=false in 619 msec 2024-11-13T03:32:54,745 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be., hostname=8394b182524b,38391,1731468755599, seqNum=18] 2024-11-13T03:32:54,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 414), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 193) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2746 (was 2808) 2024-11-13T03:32:54,762 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T03:32:54,776 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=2745 2024-11-13T03:32:54,776 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T03:32:54,794 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:54,796 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:54,796 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:54,799 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21053543, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-21053543, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:54,811 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21053543/hregion-21053543.1731468774799, exclude list is [], retry=0 2024-11-13T03:32:54,813 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:54,814 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:54,814 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:54,839 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21053543/hregion-21053543.1731468774799 2024-11-13T03:32:54,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:54,839 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 905985d7bdfd4a534b3280e75c0a3092, NAME => 'testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741893_1071 (size=67) 2024-11-13T03:32:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741893_1071 (size=67) 2024-11-13T03:32:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741893_1071 (size=67) 2024-11-13T03:32:54,847 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:54,848 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,850 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName a 2024-11-13T03:32:54,850 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,850 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,850 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,852 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName b 2024-11-13T03:32:54,852 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,852 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,852 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,853 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName c 2024-11-13T03:32:54,853 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,854 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,854 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,854 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,856 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,856 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,856 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:54,857 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,859 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:54,859 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 905985d7bdfd4a534b3280e75c0a3092; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63812368, jitterRate=-0.04912161827087402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 905985d7bdfd4a534b3280e75c0a3092: Writing region info on filesystem at 1731468774847Initializing all the Stores at 1731468774848 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774848Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774848Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774848Cleaning up temporary data from old regions at 1731468774856 (+8 ms)Region opened successfully at 1731468774860 (+4 ms) 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 905985d7bdfd4a534b3280e75c0a3092, disabling compactions & flushes 2024-11-13T03:32:54,860 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. after waiting 0 ms 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:54,860 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:54,860 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 905985d7bdfd4a534b3280e75c0a3092: Waiting for close lock at 1731468774860Disabling compacts and flushes for region at 1731468774860Disabling writes for close at 1731468774860Writing region close event to WAL at 1731468774860Closed at 1731468774860 2024-11-13T03:32:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741892_1070 (size=175) 2024-11-13T03:32:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741892_1070 (size=175) 2024-11-13T03:32:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741892_1070 (size=175) 2024-11-13T03:32:54,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:54,865 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21053543:(num 1731468774799) 2024-11-13T03:32:54,865 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:54,867 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:54,878 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, exclude list is [], retry=0 2024-11-13T03:32:54,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:54,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:54,882 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:54,907 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 2024-11-13T03:32:54,907 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:54,907 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 905985d7bdfd4a534b3280e75c0a3092, NAME => 'testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:54,907 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:54,907 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,907 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,909 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,910 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName a 2024-11-13T03:32:54,910 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,910 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,910 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,911 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName b 2024-11-13T03:32:54,911 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,912 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,912 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,913 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName c 2024-11-13T03:32:54,913 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:54,913 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:54,913 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,916 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,916 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,917 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:54,918 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:54,919 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 905985d7bdfd4a534b3280e75c0a3092; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62151807, jitterRate=-0.07386590540409088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:54,919 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 905985d7bdfd4a534b3280e75c0a3092: Writing region info on filesystem at 1731468774907Initializing all the Stores at 1731468774908 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774908Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774909 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468774909Cleaning up temporary data from old regions at 1731468774916 (+7 ms)Region opened successfully at 1731468774919 (+3 ms) 2024-11-13T03:32:54,944 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 905985d7bdfd4a534b3280e75c0a3092 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T03:32:54,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/a/5bc814f15e2b4e49a942044f246e6176 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731468774919/Put/seqid=0 2024-11-13T03:32:54,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741895_1073 (size=5958) 2024-11-13T03:32:54,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741895_1073 (size=5958) 2024-11-13T03:32:54,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741895_1073 (size=5958) 2024-11-13T03:32:54,971 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/a/5bc814f15e2b4e49a942044f246e6176 2024-11-13T03:32:54,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/b24dff1fe19d4688a6ed06a6a97560ef is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468774926/Put/seqid=0 2024-11-13T03:32:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741896_1074 (size=5958) 2024-11-13T03:32:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741896_1074 (size=5958) 2024-11-13T03:32:54,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741896_1074 (size=5958) 2024-11-13T03:32:54,997 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/b24dff1fe19d4688a6ed06a6a97560ef 2024-11-13T03:32:55,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/c/3b199acab46b41d8a599ed79d4b2e0da is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731468774933/Put/seqid=0 2024-11-13T03:32:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741897_1075 (size=5958) 2024-11-13T03:32:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741897_1075 (size=5958) 2024-11-13T03:32:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741897_1075 (size=5958) 2024-11-13T03:32:55,024 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/c/3b199acab46b41d8a599ed79d4b2e0da 2024-11-13T03:32:55,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/a/5bc814f15e2b4e49a942044f246e6176 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/a/5bc814f15e2b4e49a942044f246e6176 2024-11-13T03:32:55,034 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/a/5bc814f15e2b4e49a942044f246e6176, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:55,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/b24dff1fe19d4688a6ed06a6a97560ef as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/b24dff1fe19d4688a6ed06a6a97560ef 2024-11-13T03:32:55,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/b24dff1fe19d4688a6ed06a6a97560ef, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:55,042 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/c/3b199acab46b41d8a599ed79d4b2e0da as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/c/3b199acab46b41d8a599ed79d4b2e0da 2024-11-13T03:32:55,048 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/c/3b199acab46b41d8a599ed79d4b2e0da, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:55,050 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 905985d7bdfd4a534b3280e75c0a3092 in 105ms, sequenceid=33, compaction requested=false 2024-11-13T03:32:55,050 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 905985d7bdfd4a534b3280e75c0a3092: 2024-11-13T03:32:55,050 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 905985d7bdfd4a534b3280e75c0a3092, disabling compactions & flushes 2024-11-13T03:32:55,050 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:55,050 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:55,050 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. after waiting 0 ms 2024-11-13T03:32:55,050 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:55,051 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. 2024-11-13T03:32:55,052 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 905985d7bdfd4a534b3280e75c0a3092: Waiting for close lock at 1731468775050Disabling compacts and flushes for region at 1731468775050Disabling writes for close at 1731468775050Writing region close event to WAL at 1731468775051 (+1 ms)Closed at 1731468775051 2024-11-13T03:32:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741894_1072 (size=6311) 2024-11-13T03:32:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741894_1072 (size=6311) 2024-11-13T03:32:55,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741894_1072 (size=6311) 2024-11-13T03:32:55,059 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/b24dff1fe19d4688a6ed06a6a97560ef to hdfs://localhost:33331/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/b24dff1fe19d4688a6ed06a6a97560ef 2024-11-13T03:32:55,074 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, size=6.2 K (6311bytes) 2024-11-13T03:32:55,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 2024-11-13T03:32:55,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 after 0ms 2024-11-13T03:32:55,099 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:55,100 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 took 26ms 2024-11-13T03:32:55,102 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 so closing down 2024-11-13T03:32:55,102 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:55,103 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468774867.temp 2024-11-13T03:32:55,127 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp 2024-11-13T03:32:55,127 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:55,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741898_1076 (size=5575) 2024-11-13T03:32:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741898_1076 (size=5575) 2024-11-13T03:32:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741898_1076 (size=5575) 2024-11-13T03:32:55,133 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:55,134 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 2024-11-13T03:32:55,135 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-13T03:32:55,135 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, journal: Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, size=6.2 K (6311bytes) at 1731468775074Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 so closing down at 1731468775102 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp at 1731468775127 (+25 ms)3 split writer threads finished at 1731468775127Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468775133 (+6 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000003-wal.1731468774867.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 at 1731468775134 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731468775135 (+1 ms) 2024-11-13T03:32:55,136 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468774867 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468774867 2024-11-13T03:32:55,137 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 2024-11-13T03:32:55,137 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:55,139 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:55,150 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468775139, exclude list is [], retry=0 2024-11-13T03:32:55,152 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:55,153 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:55,153 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:55,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T03:32:55,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:55,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T03:32:55,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:55,175 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T03:32:55,175 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:55,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468775139 2024-11-13T03:32:55,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:32:55,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 905985d7bdfd4a534b3280e75c0a3092, NAME => 'testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:55,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:55,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,177 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,178 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,179 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName a 2024-11-13T03:32:55,179 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,184 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/a/5bc814f15e2b4e49a942044f246e6176 2024-11-13T03:32:55,185 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,185 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,186 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName b 2024-11-13T03:32:55,187 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,187 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,187 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,188 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 905985d7bdfd4a534b3280e75c0a3092 columnFamilyName c 2024-11-13T03:32:55,188 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,194 DEBUG [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/c/3b199acab46b41d8a599ed79d4b2e0da 2024-11-13T03:32:55,194 INFO [StoreOpener-905985d7bdfd4a534b3280e75c0a3092-1 {}] regionserver.HStore(327): Store=905985d7bdfd4a534b3280e75c0a3092/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,194 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,197 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 2024-11-13T03:32:55,223 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:32:55,225 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 2024-11-13T03:32:55,225 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 905985d7bdfd4a534b3280e75c0a3092 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T03:32:55,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/6a6be6d820534720ae6db41521cfacab is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468774926/Put/seqid=0 2024-11-13T03:32:55,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:55,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:55,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:55,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/6a6be6d820534720ae6db41521cfacab 2024-11-13T03:32:55,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/.tmp/b/6a6be6d820534720ae6db41521cfacab as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/6a6be6d820534720ae6db41521cfacab 2024-11-13T03:32:55,256 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/b/6a6be6d820534720ae6db41521cfacab, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T03:32:55,257 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 905985d7bdfd4a534b3280e75c0a3092 in 31ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T03:32:55,257 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/0000000000000000032 2024-11-13T03:32:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,260 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:55,261 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 905985d7bdfd4a534b3280e75c0a3092 2024-11-13T03:32:55,263 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/905985d7bdfd4a534b3280e75c0a3092/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T03:32:55,265 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 905985d7bdfd4a534b3280e75c0a3092; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72518303, jitterRate=0.08060692250728607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 905985d7bdfd4a534b3280e75c0a3092: Writing region info on filesystem at 1731468775177Initializing all the Stores at 1731468775178 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775178Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775178Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775178Obtaining lock to block concurrent updates at 1731468775225 (+47 ms)Preparing flush snapshotting stores in 905985d7bdfd4a534b3280e75c0a3092 at 1731468775225Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731468775225Flushing stores of testReplayEditsWrittenViaHRegion,,1731468774795.905985d7bdfd4a534b3280e75c0a3092. at 1731468775225Flushing 905985d7bdfd4a534b3280e75c0a3092/b: creating writer at 1731468775225Flushing 905985d7bdfd4a534b3280e75c0a3092/b: appending metadata at 1731468775237 (+12 ms)Flushing 905985d7bdfd4a534b3280e75c0a3092/b: closing flushed file at 1731468775237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb1d856: reopening flushed file at 1731468775249 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 905985d7bdfd4a534b3280e75c0a3092 in 31ms, sequenceid=32, compaction requested=false; wal=null at 1731468775257 (+8 ms)Cleaning up temporary data from old regions at 1731468775259 (+2 ms)Region opened successfully at 1731468775265 (+6 ms) 2024-11-13T03:32:55,286 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:43432 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:47490 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:43220 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:49986 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:52268 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:45754 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 202), ProcessCount=11 (was 11), AvailableMemoryMB=2740 (was 2745) 2024-11-13T03:32:55,286 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T03:32:55,297 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=2739 2024-11-13T03:32:55,298 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T03:32:55,316 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:55,318 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:55,319 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:55,321 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24627055, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-24627055, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:55,333 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24627055/hregion-24627055.1731468775322, exclude list is [], retry=0 2024-11-13T03:32:55,337 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:55,337 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:55,338 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:55,362 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24627055/hregion-24627055.1731468775322 2024-11-13T03:32:55,362 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:55,362 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b05eeae38e12959f0a34d2498b34f0e2, NAME => 'testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:32:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741902_1080 (size=68) 2024-11-13T03:32:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741902_1080 (size=68) 2024-11-13T03:32:55,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741902_1080 (size=68) 2024-11-13T03:32:55,373 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:55,374 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,375 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName a 2024-11-13T03:32:55,376 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,376 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,376 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,377 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName b 2024-11-13T03:32:55,378 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,378 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,378 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,379 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName c 2024-11-13T03:32:55,379 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,380 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,380 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,381 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,381 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,382 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,382 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,382 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:55,383 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,386 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:55,386 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b05eeae38e12959f0a34d2498b34f0e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70817585, jitterRate=0.055264249444007874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b05eeae38e12959f0a34d2498b34f0e2: Writing region info on filesystem at 1731468775373Initializing all the Stores at 1731468775374 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775374Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775374Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775374Cleaning up temporary data from old regions at 1731468775382 (+8 ms)Region opened successfully at 1731468775387 (+5 ms) 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b05eeae38e12959f0a34d2498b34f0e2, disabling compactions & flushes 2024-11-13T03:32:55,387 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. after waiting 0 ms 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:32:55,387 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:32:55,387 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b05eeae38e12959f0a34d2498b34f0e2: Waiting for close lock at 1731468775387Disabling compacts and flushes for region at 1731468775387Disabling writes for close at 1731468775387Writing region close event to WAL at 1731468775387Closed at 1731468775387 2024-11-13T03:32:55,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741901_1079 (size=175) 2024-11-13T03:32:55,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741901_1079 (size=175) 2024-11-13T03:32:55,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741901_1079 (size=175) 2024-11-13T03:32:55,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:55,397 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24627055:(num 1731468775322) 2024-11-13T03:32:55,397 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:55,399 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:55,411 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, exclude list is [], retry=0 2024-11-13T03:32:55,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:32:55,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:32:55,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:32:55,438 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 2024-11-13T03:32:55,438 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:32:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b05eeae38e12959f0a34d2498b34f0e2, NAME => 'testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:55,511 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:55,511 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,511 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,513 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,514 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName a 2024-11-13T03:32:55,514 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,515 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,515 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,516 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName b 2024-11-13T03:32:55,516 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,517 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,517 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,518 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName c 2024-11-13T03:32:55,518 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:55,518 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:55,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,522 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,522 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,522 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:55,524 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,525 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b05eeae38e12959f0a34d2498b34f0e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65327132, jitterRate=-0.026549875736236572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:55,525 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:32:55,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b05eeae38e12959f0a34d2498b34f0e2: Running coprocessor pre-open hook at 1731468775512Writing region info on filesystem at 1731468775512Initializing all the Stores at 1731468775513 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775513Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775513Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468775513Cleaning up temporary data from old regions at 1731468775522 (+9 ms)Running coprocessor post-open hooks at 1731468775525 (+3 ms)Region opened successfully at 1731468775526 (+1 ms) 2024-11-13T03:32:55,539 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b05eeae38e12959f0a34d2498b34f0e2 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T03:32:55,541 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:56,542 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:57,269 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:32:57,543 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:58,544 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:59,545 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:00,546 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:01,547 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:02,548 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:03,550 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:04,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:33:04,551 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b05eeae38e12959f0a34d2498b34f0e2/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:04,554 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b05eeae38e12959f0a34d2498b34f0e2: 2024-11-13T03:33:04,554 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,570 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b05eeae38e12959f0a34d2498b34f0e2: 2024-11-13T03:33:04,570 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T03:33:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b05eeae38e12959f0a34d2498b34f0e2, disabling compactions & flushes 2024-11-13T03:33:04,571 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. after waiting 0 ms 2024-11-13T03:33:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,571 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,571 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. 2024-11-13T03:33:04,571 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b05eeae38e12959f0a34d2498b34f0e2: Waiting for close lock at 1731468784571Running coprocessor pre-close hooks at 1731468784571Disabling compacts and flushes for region at 1731468784571Disabling writes for close at 1731468784571Writing region close event to WAL at 1731468784571Running coprocessor post-close hooks at 1731468784571Closed at 1731468784571 2024-11-13T03:33:04,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741903_1081 (size=4236) 2024-11-13T03:33:04,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741903_1081 (size=4236) 2024-11-13T03:33:04,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741903_1081 (size=4236) 2024-11-13T03:33:04,588 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, size=4.1 K (4236bytes) 2024-11-13T03:33:04,588 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 2024-11-13T03:33:04,589 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 after 1ms 2024-11-13T03:33:04,614 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:04,614 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 took 26ms 2024-11-13T03:33:04,617 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 so closing down 2024-11-13T03:33:04,617 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:33:04,618 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731468775400.temp 2024-11-13T03:33:04,647 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp 2024-11-13T03:33:04,647 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:33:04,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741904_1082 (size=3245) 2024-11-13T03:33:04,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741904_1082 (size=3245) 2024-11-13T03:33:04,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741904_1082 (size=3245) 2024-11-13T03:33:04,654 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T03:33:04,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 2024-11-13T03:33:04,655 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-13T03:33:04,655 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, journal: Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, size=4.1 K (4236bytes) at 1731468784588Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 so closing down at 1731468784617 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp at 1731468784647 (+30 ms)3 split writer threads finished at 1731468784647Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731468784654 (+7 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000004-wal.1731468775400.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 at 1731468784655 (+1 ms)Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731468784655 2024-11-13T03:33:04,657 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468775400 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468775400 2024-11-13T03:33:04,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 2024-11-13T03:33:04,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:04,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:04,674 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468784659, exclude list is [], retry=0 2024-11-13T03:33:04,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:04,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:04,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:04,702 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468784659 2024-11-13T03:33:04,702 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:33:04,702 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b05eeae38e12959f0a34d2498b34f0e2, NAME => 'testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:04,703 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,703 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:04,703 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,703 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,705 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,706 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName a 2024-11-13T03:33:04,706 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:04,707 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:04,707 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,708 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName b 2024-11-13T03:33:04,708 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:04,708 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:04,708 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,709 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b05eeae38e12959f0a34d2498b34f0e2 columnFamilyName c 2024-11-13T03:33:04,709 DEBUG [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:04,710 INFO [StoreOpener-b05eeae38e12959f0a34d2498b34f0e2-1 {}] regionserver.HStore(327): Store=b05eeae38e12959f0a34d2498b34f0e2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:04,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,713 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 2024-11-13T03:33:04,739 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:04,740 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 2024-11-13T03:33:04,741 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b05eeae38e12959f0a34d2498b34f0e2 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T03:33:04,755 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/a/3495d51a5ac045809cf136f2904f8a18 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731468784561/Put/seqid=0 2024-11-13T03:33:04,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741906_1084 (size=5523) 2024-11-13T03:33:04,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741906_1084 (size=5523) 2024-11-13T03:33:04,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741906_1084 (size=5523) 2024-11-13T03:33:04,762 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/a/3495d51a5ac045809cf136f2904f8a18 2024-11-13T03:33:04,787 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/b/cc84e46c9ee64bfb8a7dc4435af5295f is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731468784556/Put/seqid=0 2024-11-13T03:33:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741907_1085 (size=5524) 2024-11-13T03:33:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741907_1085 (size=5524) 2024-11-13T03:33:04,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741907_1085 (size=5524) 2024-11-13T03:33:04,794 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/b/cc84e46c9ee64bfb8a7dc4435af5295f 2024-11-13T03:33:04,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/c/34c3452d977d40ec92fa04a9847727b5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731468784559/Put/seqid=0 2024-11-13T03:33:04,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741908_1086 (size=5457) 2024-11-13T03:33:04,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741908_1086 (size=5457) 2024-11-13T03:33:04,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741908_1086 (size=5457) 2024-11-13T03:33:04,829 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/c/34c3452d977d40ec92fa04a9847727b5 2024-11-13T03:33:04,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/a/3495d51a5ac045809cf136f2904f8a18 as hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/a/3495d51a5ac045809cf136f2904f8a18 2024-11-13T03:33:04,841 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/a/3495d51a5ac045809cf136f2904f8a18, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T03:33:04,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/b/cc84e46c9ee64bfb8a7dc4435af5295f as hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/b/cc84e46c9ee64bfb8a7dc4435af5295f 2024-11-13T03:33:04,848 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/b/cc84e46c9ee64bfb8a7dc4435af5295f, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T03:33:04,849 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/.tmp/c/34c3452d977d40ec92fa04a9847727b5 as hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/c/34c3452d977d40ec92fa04a9847727b5 2024-11-13T03:33:04,855 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/c/34c3452d977d40ec92fa04a9847727b5, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T03:33:04,855 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b05eeae38e12959f0a34d2498b34f0e2 in 114ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T03:33:04,856 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/0000000000000000026 2024-11-13T03:33:04,857 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,857 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,858 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:04,859 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,861 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsAfterAbortingFlush/b05eeae38e12959f0a34d2498b34f0e2/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T03:33:04,862 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b05eeae38e12959f0a34d2498b34f0e2; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59544758, jitterRate=-0.11271396279335022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:04,863 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b05eeae38e12959f0a34d2498b34f0e2 2024-11-13T03:33:04,863 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b05eeae38e12959f0a34d2498b34f0e2: Running coprocessor pre-open hook at 1731468784703Writing region info on filesystem at 1731468784703Initializing all the Stores at 1731468784704 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468784704Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468784705 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468784705Obtaining lock to block concurrent updates at 1731468784741 (+36 ms)Preparing flush snapshotting stores in b05eeae38e12959f0a34d2498b34f0e2 at 1731468784741Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731468784741Flushing stores of testReplayEditsAfterAbortingFlush,,1731468775317.b05eeae38e12959f0a34d2498b34f0e2. at 1731468784741Flushing b05eeae38e12959f0a34d2498b34f0e2/a: creating writer at 1731468784741Flushing b05eeae38e12959f0a34d2498b34f0e2/a: appending metadata at 1731468784755 (+14 ms)Flushing b05eeae38e12959f0a34d2498b34f0e2/a: closing flushed file at 1731468784755Flushing b05eeae38e12959f0a34d2498b34f0e2/b: creating writer at 1731468784767 (+12 ms)Flushing b05eeae38e12959f0a34d2498b34f0e2/b: appending metadata at 1731468784786 (+19 ms)Flushing b05eeae38e12959f0a34d2498b34f0e2/b: closing flushed file at 1731468784786Flushing b05eeae38e12959f0a34d2498b34f0e2/c: creating writer at 1731468784801 (+15 ms)Flushing b05eeae38e12959f0a34d2498b34f0e2/c: appending metadata at 1731468784821 (+20 ms)Flushing b05eeae38e12959f0a34d2498b34f0e2/c: closing flushed file at 1731468784821Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d4d0cf5: reopening flushed file at 1731468784835 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73d9a4c8: reopening flushed file at 1731468784842 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294d0f85: reopening flushed file at 1731468784848 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b05eeae38e12959f0a34d2498b34f0e2 in 114ms, sequenceid=26, compaction requested=false; wal=null at 1731468784855 (+7 ms)Cleaning up temporary data from old regions at 1731468784857 (+2 ms)Running coprocessor post-open hooks at 1731468784863 (+6 ms)Region opened successfully at 1731468784863 2024-11-13T03:33:04,884 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=416 (was 418), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 202), ProcessCount=11 (was 11), AvailableMemoryMB=2763 (was 2739) - AvailableMemoryMB LEAK? - 2024-11-13T03:33:04,885 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-13T03:33:04,896 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=416, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=2763 2024-11-13T03:33:04,896 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-13T03:33:04,910 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:04,912 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:04,913 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:33:04,915 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-40413836, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-40413836, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:04,927 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-40413836/hregion-40413836.1731468784916, exclude list is [], retry=0 2024-11-13T03:33:04,930 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:04,930 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:04,930 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:04,954 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-40413836/hregion-40413836.1731468784916 2024-11-13T03:33:04,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:33:04,954 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2215e76dd4ce6e518d2d150e5b2c29dc, NAME => 'testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:33:04,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741910_1088 (size=61) 2024-11-13T03:33:04,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741910_1088 (size=61) 2024-11-13T03:33:04,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741910_1088 (size=61) 2024-11-13T03:33:04,963 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:04,964 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,966 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2215e76dd4ce6e518d2d150e5b2c29dc columnFamilyName a 2024-11-13T03:33:04,966 DEBUG [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:04,966 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(327): Store=2215e76dd4ce6e518d2d150e5b2c29dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:04,966 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,967 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,967 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,969 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:04,971 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:33:04,971 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2215e76dd4ce6e518d2d150e5b2c29dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64974009, jitterRate=-0.03181181848049164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:33:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Writing region info on filesystem at 1731468784963Initializing all the Stores at 1731468784964 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468784964Cleaning up temporary data from old regions at 1731468784967 (+3 ms)Region opened successfully at 1731468784971 (+4 ms) 2024-11-13T03:33:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2215e76dd4ce6e518d2d150e5b2c29dc, disabling compactions & flushes 2024-11-13T03:33:04,972 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. after waiting 0 ms 2024-11-13T03:33:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:04,972 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Waiting for close lock at 1731468784972Disabling compacts and flushes for region at 1731468784972Disabling writes for close at 1731468784972Writing region close event to WAL at 1731468784972Closed at 1731468784972 2024-11-13T03:33:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741909_1087 (size=175) 2024-11-13T03:33:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741909_1087 (size=175) 2024-11-13T03:33:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741909_1087 (size=175) 2024-11-13T03:33:04,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:33:04,976 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-40413836:(num 1731468784916) 2024-11-13T03:33:04,977 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:04,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:04,990 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, exclude list is [], retry=0 2024-11-13T03:33:04,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:04,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:04,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:05,017 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 2024-11-13T03:33:05,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:33:05,018 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2215e76dd4ce6e518d2d150e5b2c29dc, NAME => 'testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:05,018 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:05,018 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,018 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,020 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,021 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2215e76dd4ce6e518d2d150e5b2c29dc columnFamilyName a 2024-11-13T03:33:05,021 DEBUG [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,021 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(327): Store=2215e76dd4ce6e518d2d150e5b2c29dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,022 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,026 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2215e76dd4ce6e518d2d150e5b2c29dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74993930, jitterRate=0.11749663949012756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:33:05,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Writing region info on filesystem at 1731468785018Initializing all the Stores at 1731468785019 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785020 (+1 ms)Cleaning up temporary data from old regions at 1731468785024 (+4 ms)Region opened successfully at 1731468785028 (+4 ms) 2024-11-13T03:33:05,038 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2215e76dd4ce6e518d2d150e5b2c29dc, disabling compactions & flushes 2024-11-13T03:33:05,038 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:05,038 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:05,039 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. after waiting 0 ms 2024-11-13T03:33:05,039 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:05,039 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:05,039 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. 2024-11-13T03:33:05,039 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Waiting for close lock at 1731468785038Disabling compacts and flushes for region at 1731468785038Disabling writes for close at 1731468785039 (+1 ms)Writing region close event to WAL at 1731468785039Closed at 1731468785039 2024-11-13T03:33:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741911_1089 (size=1795) 2024-11-13T03:33:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741911_1089 (size=1795) 2024-11-13T03:33:05,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741911_1089 (size=1795) 2024-11-13T03:33:05,060 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, size=1.8 K (1795bytes) 2024-11-13T03:33:05,060 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 2024-11-13T03:33:05,061 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 after 0ms 2024-11-13T03:33:05,087 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:05,087 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 took 27ms 2024-11-13T03:33:05,089 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 so closing down 2024-11-13T03:33:05,089 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:33:05,090 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468784978.temp 2024-11-13T03:33:05,115 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp 2024-11-13T03:33:05,115 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:33:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741912_1090 (size=1795) 2024-11-13T03:33:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741912_1090 (size=1795) 2024-11-13T03:33:05,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741912_1090 (size=1795) 2024-11-13T03:33:05,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T03:33:05,123 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp to hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 2024-11-13T03:33:05,124 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-13T03:33:05,124 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, journal: Splitting hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, size=1.8 K (1795bytes) at 1731468785060Finishing writing output for hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 so closing down at 1731468785089 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp at 1731468785115 (+26 ms)3 split writer threads finished at 1731468785115Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731468785122 (+7 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000003-wal.1731468784978.temp to hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 at 1731468785123 (+1 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731468785124 (+1 ms) 2024-11-13T03:33:05,125 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468784978 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468784978 2024-11-13T03:33:05,126 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 2024-11-13T03:33:05,152 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:05,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T03:33:05,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T03:33:05,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T03:33:05,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T03:33:05,452 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:05,454 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:05,466 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468785454, exclude list is [], retry=0 2024-11-13T03:33:05,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:05,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:05,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:05,545 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468785454 2024-11-13T03:33:05,546 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:33:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2215e76dd4ce6e518d2d150e5b2c29dc, NAME => 'testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,546 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,549 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,550 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2215e76dd4ce6e518d2d150e5b2c29dc columnFamilyName a 2024-11-13T03:33:05,550 DEBUG [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,551 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(327): Store=2215e76dd4ce6e518d2d150e5b2c29dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,554 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 2024-11-13T03:33:05,580 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:05,581 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 2024-11-13T03:33:05,581 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2215e76dd4ce6e518d2d150e5b2c29dc 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T03:33:05,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/.tmp/a/40617f7b050241aba7660a4c69ef3ebf is 79, key is testDatalossWhenInputError/a:x0/1731468785028/Put/seqid=0 2024-11-13T03:33:05,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741914_1092 (size=5808) 2024-11-13T03:33:05,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741914_1092 (size=5808) 2024-11-13T03:33:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741914_1092 (size=5808) 2024-11-13T03:33:05,607 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/.tmp/a/40617f7b050241aba7660a4c69ef3ebf 2024-11-13T03:33:05,617 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/.tmp/a/40617f7b050241aba7660a4c69ef3ebf as hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/a/40617f7b050241aba7660a4c69ef3ebf 2024-11-13T03:33:05,626 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/a/40617f7b050241aba7660a4c69ef3ebf, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T03:33:05,626 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 2215e76dd4ce6e518d2d150e5b2c29dc in 45ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T03:33:05,627 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/0000000000000000012 2024-11-13T03:33:05,628 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,628 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,631 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,633 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T03:33:05,634 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2215e76dd4ce6e518d2d150e5b2c29dc; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71380443, jitterRate=0.06365148723125458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:33:05,635 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Writing region info on filesystem at 1731468785546Initializing all the Stores at 1731468785548 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785548Obtaining lock to block concurrent updates at 1731468785581 (+33 ms)Preparing flush snapshotting stores in 2215e76dd4ce6e518d2d150e5b2c29dc at 1731468785581Finished memstore snapshotting testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731468785581Flushing stores of testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc. at 1731468785581Flushing 2215e76dd4ce6e518d2d150e5b2c29dc/a: creating writer at 1731468785581Flushing 2215e76dd4ce6e518d2d150e5b2c29dc/a: appending metadata at 1731468785598 (+17 ms)Flushing 2215e76dd4ce6e518d2d150e5b2c29dc/a: closing flushed file at 1731468785598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a6fc6df: reopening flushed file at 1731468785615 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 2215e76dd4ce6e518d2d150e5b2c29dc in 45ms, sequenceid=12, compaction requested=false; wal=null at 1731468785626 (+11 ms)Cleaning up temporary data from old regions at 1731468785628 (+2 ms)Region opened successfully at 1731468785635 (+7 ms) 2024-11-13T03:33:05,638 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2215e76dd4ce6e518d2d150e5b2c29dc, NAME => 'testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:05,638 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468784910.2215e76dd4ce6e518d2d150e5b2c29dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:05,639 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,639 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,640 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,641 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2215e76dd4ce6e518d2d150e5b2c29dc columnFamilyName a 2024-11-13T03:33:05,641 DEBUG [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,646 DEBUG [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/a/40617f7b050241aba7660a4c69ef3ebf 2024-11-13T03:33:05,646 INFO [StoreOpener-2215e76dd4ce6e518d2d150e5b2c29dc-1 {}] regionserver.HStore(327): Store=2215e76dd4ce6e518d2d150e5b2c29dc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,647 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,647 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,649 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,650 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,650 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,653 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2215e76dd4ce6e518d2d150e5b2c29dc 2024-11-13T03:33:05,656 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testDatalossWhenInputError/2215e76dd4ce6e518d2d150e5b2c29dc/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T03:33:05,657 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2215e76dd4ce6e518d2d150e5b2c29dc; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65428494, jitterRate=-0.025039464235305786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:33:05,657 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2215e76dd4ce6e518d2d150e5b2c29dc: Writing region info on filesystem at 1731468785639Initializing all the Stores at 1731468785639Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785640 (+1 ms)Cleaning up temporary data from old regions at 1731468785650 (+10 ms)Region opened successfully at 1731468785657 (+7 ms) 2024-11-13T03:33:05,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 416) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:54106 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:35092 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:33478 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:54008 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:35040 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:33532 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 171), ProcessCount=11 (was 11), AvailableMemoryMB=2757 (was 2763) 2024-11-13T03:33:05,674 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T03:33:05,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=2756 2024-11-13T03:33:05,686 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T03:33:05,700 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:05,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:05,702 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:33:05,704 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02053362, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-02053362, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:05,715 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02053362/hregion-02053362.1731468785705, exclude list is [], retry=0 2024-11-13T03:33:05,718 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:05,718 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:05,718 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:05,764 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02053362/hregion-02053362.1731468785705 2024-11-13T03:33:05,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:33:05,765 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7435a572012a82d4405a494cc8ae2f5e, NAME => 'testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:33:05,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741916_1094 (size=63) 2024-11-13T03:33:05,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741916_1094 (size=63) 2024-11-13T03:33:05,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741916_1094 (size=63) 2024-11-13T03:33:05,775 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:05,776 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,777 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName a 2024-11-13T03:33:05,777 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,778 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,778 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,779 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName b 2024-11-13T03:33:05,779 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,780 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,780 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,781 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName c 2024-11-13T03:33:05,781 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,782 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,782 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,784 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,784 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,785 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:05,786 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,788 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:33:05,788 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7435a572012a82d4405a494cc8ae2f5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63852393, jitterRate=-0.04852519929409027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7435a572012a82d4405a494cc8ae2f5e: Writing region info on filesystem at 1731468785775Initializing all the Stores at 1731468785775Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785775Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785776 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785776Cleaning up temporary data from old regions at 1731468785784 (+8 ms)Region opened successfully at 1731468785789 (+5 ms) 2024-11-13T03:33:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7435a572012a82d4405a494cc8ae2f5e, disabling compactions & flushes 2024-11-13T03:33:05,789 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. after waiting 0 ms 2024-11-13T03:33:05,789 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:05,790 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:05,790 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7435a572012a82d4405a494cc8ae2f5e: Waiting for close lock at 1731468785789Disabling compacts and flushes for region at 1731468785789Disabling writes for close at 1731468785789Writing region close event to WAL at 1731468785790 (+1 ms)Closed at 1731468785790 2024-11-13T03:33:05,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741915_1093 (size=175) 2024-11-13T03:33:05,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741915_1093 (size=175) 2024-11-13T03:33:05,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741915_1093 (size=175) 2024-11-13T03:33:05,794 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:33:05,795 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02053362:(num 1731468785705) 2024-11-13T03:33:05,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:05,796 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:05,809 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, exclude list is [], retry=0 2024-11-13T03:33:05,811 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:05,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:05,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:05,839 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 2024-11-13T03:33:05,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:33:05,840 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7435a572012a82d4405a494cc8ae2f5e, NAME => 'testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:05,840 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:05,840 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,840 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,842 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,843 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName a 2024-11-13T03:33:05,843 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,843 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,844 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,845 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName b 2024-11-13T03:33:05,845 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,845 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,845 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,846 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName c 2024-11-13T03:33:05,846 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:05,847 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:05,847 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,847 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,850 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,850 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,850 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:05,851 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:05,852 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7435a572012a82d4405a494cc8ae2f5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71772181, jitterRate=0.06948883831501007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:05,853 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7435a572012a82d4405a494cc8ae2f5e: Writing region info on filesystem at 1731468785840Initializing all the Stores at 1731468785841 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785841Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785842 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468785842Cleaning up temporary data from old regions at 1731468785850 (+8 ms)Region opened successfully at 1731468785853 (+3 ms) 2024-11-13T03:33:05,857 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731468785856/Put/seqid=0 2024-11-13T03:33:05,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741918_1096 (size=4875) 2024-11-13T03:33:05,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741918_1096 (size=4875) 2024-11-13T03:33:05,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741918_1096 (size=4875) 2024-11-13T03:33:05,865 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731468785865/Put/seqid=0 2024-11-13T03:33:05,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741919_1097 (size=4875) 2024-11-13T03:33:05,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741919_1097 (size=4875) 2024-11-13T03:33:05,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741919_1097 (size=4875) 2024-11-13T03:33:05,877 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731468785877/Put/seqid=0 2024-11-13T03:33:05,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741920_1098 (size=4875) 2024-11-13T03:33:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741920_1098 (size=4875) 2024-11-13T03:33:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741920_1098 (size=4875) 2024-11-13T03:33:05,888 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,893 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T03:33:05,893 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:33:05,893 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,897 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T03:33:05,897 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:33:05,897 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,900 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T03:33:05,900 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:33:05,901 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7435a572012a82d4405a494cc8ae2f5e 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T03:33:05,916 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp/a/895e1662a4c94756934a6641e17f73e6 is 55, key is testCompactedBulkLoadedFiles/a:a/1731468785853/Put/seqid=0 2024-11-13T03:33:05,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741921_1099 (size=5107) 2024-11-13T03:33:05,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741921_1099 (size=5107) 2024-11-13T03:33:05,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741921_1099 (size=5107) 2024-11-13T03:33:05,924 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp/a/895e1662a4c94756934a6641e17f73e6 2024-11-13T03:33:05,931 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp/a/895e1662a4c94756934a6641e17f73e6 as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6 2024-11-13T03:33:05,938 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T03:33:05,939 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 7435a572012a82d4405a494cc8ae2f5e in 38ms, sequenceid=4, compaction requested=false 2024-11-13T03:33:05,939 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7435a572012a82d4405a494cc8ae2f5e: 2024-11-13T03:33:05,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ 2024-11-13T03:33:05,942 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ 2024-11-13T03:33:05,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ 2024-11-13T03:33:05,943 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile0 into 7435a572012a82d4405a494cc8ae2f5e/a as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ - updating store file list. 2024-11-13T03:33:05,948 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 46c0a8f84ed745bba511cb12a8964f74_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:05,948 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ into 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,948 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile0 into 7435a572012a82d4405a494cc8ae2f5e/a (new location: hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_) 2024-11-13T03:33:05,949 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile1 into 7435a572012a82d4405a494cc8ae2f5e/a as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ - updating store file list. 2024-11-13T03:33:05,954 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:05,954 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ into 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,954 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile1 into 7435a572012a82d4405a494cc8ae2f5e/a (new location: hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_) 2024-11-13T03:33:05,955 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile2 into 7435a572012a82d4405a494cc8ae2f5e/a as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ - updating store file list. 2024-11-13T03:33:05,959 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7e9f7c94fc854ef49a890eb21449c849_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:05,959 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ into 7435a572012a82d4405a494cc8ae2f5e/a 2024-11-13T03:33:05,959 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33331/hbase/testCompactedBulkLoadedFiles/hfile2 into 7435a572012a82d4405a494cc8ae2f5e/a (new location: hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_) 2024-11-13T03:33:05,967 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:33:05,967 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7435a572012a82d4405a494cc8ae2f5e/a is initiating major compaction (all files) 2024-11-13T03:33:05,967 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7435a572012a82d4405a494cc8ae2f5e/a in testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:05,968 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_] into tmpdir=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp, totalSize=19.3 K 2024-11-13T03:33:05,968 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 895e1662a4c94756934a6641e17f73e6, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731468785853 2024-11-13T03:33:05,969 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 46c0a8f84ed745bba511cb12a8964f74_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:33:05,969 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:33:05,970 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7e9f7c94fc854ef49a890eb21449c849_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:33:05,982 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp/a/80cf52023ee64e30840331c8d3ae6cb6 is 55, key is testCompactedBulkLoadedFiles/a:a/1731468785853/Put/seqid=0 2024-11-13T03:33:05,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741922_1100 (size=6154) 2024-11-13T03:33:05,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741922_1100 (size=6154) 2024-11-13T03:33:05,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741922_1100 (size=6154) 2024-11-13T03:33:05,999 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/.tmp/a/80cf52023ee64e30840331c8d3ae6cb6 as hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/80cf52023ee64e30840331c8d3ae6cb6 2024-11-13T03:33:06,006 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 7435a572012a82d4405a494cc8ae2f5e/a of 7435a572012a82d4405a494cc8ae2f5e into 80cf52023ee64e30840331c8d3ae6cb6(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:33:06,006 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7435a572012a82d4405a494cc8ae2f5e: 2024-11-13T03:33:06,007 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T03:33:06,007 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T03:33:06,040 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, size=0 (0bytes) 2024-11-13T03:33:06,040 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 might be still open, length is 0 2024-11-13T03:33:06,040 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 2024-11-13T03:33:06,041 WARN [IPC Server handler 3 on default port 33331 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-13T03:33:06,041 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 after 1ms 2024-11-13T03:33:06,941 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:54148 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41693:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54148 dst: /127.0.0.1:41693 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41693 remote=/127.0.0.1:54148]. Total timeout mills is 60000, 59066 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:06,942 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:35122 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35122 dst: /127.0.0.1:39941 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:06,942 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:33560 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36333:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33560 dst: /127.0.0.1:36333 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:06,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741917_1101 (size=1794) 2024-11-13T03:33:06,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741917_1101 (size=1794) 2024-11-13T03:33:10,043 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 after 4002ms 2024-11-13T03:33:10,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:10,073 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 took 4033ms 2024-11-13T03:33:10,075 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797; continuing. 2024-11-13T03:33:10,075 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 so closing down 2024-11-13T03:33:10,075 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:33:10,077 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468785797.temp 2024-11-13T03:33:10,098 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:33:10,103 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp 2024-11-13T03:33:10,103 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:33:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741923_1102 (size=800) 2024-11-13T03:33:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741923_1102 (size=800) 2024-11-13T03:33:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741923_1102 (size=800) 2024-11-13T03:33:10,110 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:33:10,111 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp to hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 2024-11-13T03:33:10,111 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:33:10,111 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, journal: Splitting hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, size=0 (0bytes) at 1731468786040Finishing writing output for hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 so closing down at 1731468790075 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp at 1731468790103 (+28 ms)3 split writer threads finished at 1731468790103Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468790110 (+7 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000003-wal.1731468785797.temp to hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 at 1731468790111 (+1 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797, size=0, length=0, corrupted=false, cancelled=false at 1731468790111 2024-11-13T03:33:10,113 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468785797 2024-11-13T03:33:10,114 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 2024-11-13T03:33:10,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:10,116 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:10,129 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468790116, exclude list is [], retry=0 2024-11-13T03:33:10,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:10,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:10,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:10,158 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468790116 2024-11-13T03:33:10,158 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:37535:37535)] 2024-11-13T03:33:10,158 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7435a572012a82d4405a494cc8ae2f5e, NAME => 'testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:10,158 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:10,158 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,158 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,160 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,161 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName a 2024-11-13T03:33:10,161 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,167 DEBUG [StoreFileOpener-7435a572012a82d4405a494cc8ae2f5e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:10,168 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ 2024-11-13T03:33:10,173 DEBUG [StoreFileOpener-7435a572012a82d4405a494cc8ae2f5e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 46c0a8f84ed745bba511cb12a8964f74_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:10,173 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ 2024-11-13T03:33:10,177 DEBUG [StoreFileOpener-7435a572012a82d4405a494cc8ae2f5e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7e9f7c94fc854ef49a890eb21449c849_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:33:10,177 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ 2024-11-13T03:33:10,181 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/80cf52023ee64e30840331c8d3ae6cb6 2024-11-13T03:33:10,185 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6 2024-11-13T03:33:10,185 WARN [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1b0eb8b6 2024-11-13T03:33:10,185 WARN [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1b0eb8b6 2024-11-13T03:33:10,185 WARN [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1b0eb8b6 2024-11-13T03:33:10,185 WARN [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1b0eb8b6 2024-11-13T03:33:10,185 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6] to archive 2024-11-13T03:33:10,186 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:33:10,188 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ to hdfs://localhost:33331/hbase/archive/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_ 2024-11-13T03:33:10,189 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ to hdfs://localhost:33331/hbase/archive/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/46c0a8f84ed745bba511cb12a8964f74_SeqId_4_ 2024-11-13T03:33:10,190 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ to hdfs://localhost:33331/hbase/archive/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/7e9f7c94fc854ef49a890eb21449c849_SeqId_4_ 2024-11-13T03:33:10,192 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6 to hdfs://localhost:33331/hbase/archive/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/a/895e1662a4c94756934a6641e17f73e6 2024-11-13T03:33:10,192 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,192 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,193 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName b 2024-11-13T03:33:10,193 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,193 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,193 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,194 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7435a572012a82d4405a494cc8ae2f5e columnFamilyName c 2024-11-13T03:33:10,194 DEBUG [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,195 INFO [StoreOpener-7435a572012a82d4405a494cc8ae2f5e-1 {}] regionserver.HStore(327): Store=7435a572012a82d4405a494cc8ae2f5e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,195 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,197 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 2024-11-13T03:33:10,223 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:10,225 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 7435a572012a82d4405a494cc8ae2f5e : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "7435a572012a82d4405a494cc8ae2f5e" family_name: "a" compaction_input: "895e1662a4c94756934a6641e17f73e6" compaction_input: "46c0a8f84ed745bba511cb12a8964f74_SeqId_4_" compaction_input: "16b6dc8a457343629ceeaa9e25d466a5_SeqId_4_" compaction_input: "7e9f7c94fc854ef49a890eb21449c849_SeqId_4_" compaction_output: "80cf52023ee64e30840331c8d3ae6cb6" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T03:33:10,225 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T03:33:10,226 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 2024-11-13T03:33:10,226 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/0000000000000000008 2024-11-13T03:33:10,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,228 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:10,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7435a572012a82d4405a494cc8ae2f5e 2024-11-13T03:33:10,232 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testCompactedBulkLoadedFiles/7435a572012a82d4405a494cc8ae2f5e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T03:33:10,233 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7435a572012a82d4405a494cc8ae2f5e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71264379, jitterRate=0.06192199885845184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:10,233 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7435a572012a82d4405a494cc8ae2f5e: Writing region info on filesystem at 1731468790159Initializing all the Stores at 1731468790159Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790159Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790160 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790160Cleaning up temporary data from old regions at 1731468790228 (+68 ms)Region opened successfully at 1731468790233 (+5 ms) 2024-11-13T03:33:10,235 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7435a572012a82d4405a494cc8ae2f5e, disabling compactions & flushes 2024-11-13T03:33:10,235 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:10,235 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:10,235 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. after waiting 0 ms 2024-11-13T03:33:10,235 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:10,236 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731468785700.7435a572012a82d4405a494cc8ae2f5e. 2024-11-13T03:33:10,236 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7435a572012a82d4405a494cc8ae2f5e: Waiting for close lock at 1731468790235Disabling compacts and flushes for region at 1731468790235Disabling writes for close at 1731468790235Writing region close event to WAL at 1731468790236 (+1 ms)Closed at 1731468790236 2024-11-13T03:33:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741924_1103 (size=175) 2024-11-13T03:33:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741924_1103 (size=175) 2024-11-13T03:33:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741924_1103 (size=175) 2024-11-13T03:33:10,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:33:10,241 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468790116) 2024-11-13T03:33:10,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 425) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33331 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33331 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_958396942_22 at /127.0.0.1:35194 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_958396942_22 at /127.0.0.1:54248 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_958396942_22 at /127.0.0.1:33610 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=157 (was 171), ProcessCount=11 (was 11), AvailableMemoryMB=2728 (was 2756) 2024-11-13T03:33:10,255 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T03:33:10,269 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=157, ProcessCount=11, AvailableMemoryMB=2728 2024-11-13T03:33:10,269 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T03:33:10,282 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:10,284 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:33:10,284 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:33:10,287 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96080984, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/hregion-96080984, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:10,299 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96080984/hregion-96080984.1731468790287, exclude list is [], retry=0 2024-11-13T03:33:10,302 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:10,302 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:10,302 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:10,332 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96080984/hregion-96080984.1731468790287 2024-11-13T03:33:10,333 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:33:10,333 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => acdb77aa047cbf439e7a2da906f4a304, NAME => 'testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33331/hbase 2024-11-13T03:33:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741926_1105 (size=67) 2024-11-13T03:33:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741926_1105 (size=67) 2024-11-13T03:33:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741926_1105 (size=67) 2024-11-13T03:33:10,342 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:10,343 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,344 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName a 2024-11-13T03:33:10,344 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,345 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,345 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,346 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName b 2024-11-13T03:33:10,346 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,347 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,347 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,348 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName c 2024-11-13T03:33:10,348 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,349 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,350 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,350 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,351 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,351 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,352 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:10,353 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,355 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:33:10,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acdb77aa047cbf439e7a2da906f4a304; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68421667, jitterRate=0.019562289118766785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acdb77aa047cbf439e7a2da906f4a304: Writing region info on filesystem at 1731468790342Initializing all the Stores at 1731468790343 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790343Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790343Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790343Cleaning up temporary data from old regions at 1731468790351 (+8 ms)Region opened successfully at 1731468790356 (+5 ms) 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acdb77aa047cbf439e7a2da906f4a304, disabling compactions & flushes 2024-11-13T03:33:10,356 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. after waiting 0 ms 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,356 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acdb77aa047cbf439e7a2da906f4a304: Waiting for close lock at 1731468790356Disabling compacts and flushes for region at 1731468790356Disabling writes for close at 1731468790356Writing region close event to WAL at 1731468790356Closed at 1731468790356 2024-11-13T03:33:10,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741925_1104 (size=175) 2024-11-13T03:33:10,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741925_1104 (size=175) 2024-11-13T03:33:10,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741925_1104 (size=175) 2024-11-13T03:33:10,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:33:10,361 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96080984:(num 1731468790287) 2024-11-13T03:33:10,362 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:10,364 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:10,379 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, exclude list is [], retry=0 2024-11-13T03:33:10,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:10,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:10,382 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:10,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 2024-11-13T03:33:10,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365),(127.0.0.1/127.0.0.1:43357:43357)] 2024-11-13T03:33:10,406 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => acdb77aa047cbf439e7a2da906f4a304, NAME => 'testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:10,406 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:10,406 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,406 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,407 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,408 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName a 2024-11-13T03:33:10,408 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,409 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,409 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,410 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName b 2024-11-13T03:33:10,410 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,410 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,410 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,411 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName c 2024-11-13T03:33:10,411 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,411 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,411 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,412 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,413 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,414 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,414 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,414 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:10,416 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,416 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acdb77aa047cbf439e7a2da906f4a304; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74101942, jitterRate=0.10420498251914978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:10,416 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acdb77aa047cbf439e7a2da906f4a304: Writing region info on filesystem at 1731468790406Initializing all the Stores at 1731468790407 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790407Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790407Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790407Cleaning up temporary data from old regions at 1731468790414 (+7 ms)Region opened successfully at 1731468790416 (+2 ms) 2024-11-13T03:33:10,424 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acdb77aa047cbf439e7a2da906f4a304 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T03:33:10,437 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/40ef013a7ed04cabb5ae331ad19dab93 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731468790416/Put/seqid=0 2024-11-13T03:33:10,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741928_1107 (size=5958) 2024-11-13T03:33:10,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741928_1107 (size=5958) 2024-11-13T03:33:10,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741928_1107 (size=5958) 2024-11-13T03:33:10,444 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/40ef013a7ed04cabb5ae331ad19dab93 2024-11-13T03:33:10,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/40ef013a7ed04cabb5ae331ad19dab93 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/40ef013a7ed04cabb5ae331ad19dab93 2024-11-13T03:33:10,455 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/40ef013a7ed04cabb5ae331ad19dab93, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T03:33:10,456 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for acdb77aa047cbf439e7a2da906f4a304 in 33ms, sequenceid=13, compaction requested=false 2024-11-13T03:33:10,456 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for acdb77aa047cbf439e7a2da906f4a304: 2024-11-13T03:33:10,538 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acdb77aa047cbf439e7a2da906f4a304, disabling compactions & flushes 2024-11-13T03:33:10,538 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,538 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,538 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. after waiting 0 ms 2024-11-13T03:33:10,538 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,539 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,539 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:10,539 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acdb77aa047cbf439e7a2da906f4a304: Waiting for close lock at 1731468790538Disabling compacts and flushes for region at 1731468790538Disabling writes for close at 1731468790538Writing region close event to WAL at 1731468790539 (+1 ms)Closed at 1731468790539 2024-11-13T03:33:10,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741927_1106 (size=6243) 2024-11-13T03:33:10,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741927_1106 (size=6243) 2024-11-13T03:33:10,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741927_1106 (size=6243) 2024-11-13T03:33:10,557 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, size=6.1 K (6243bytes) 2024-11-13T03:33:10,557 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 2024-11-13T03:33:10,557 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 after 0ms 2024-11-13T03:33:10,581 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:10,581 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 took 24ms 2024-11-13T03:33:10,583 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 so closing down 2024-11-13T03:33:10,583 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:33:10,584 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468790364.temp 2024-11-13T03:33:10,608 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp 2024-11-13T03:33:10,609 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:33:10,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741929_1108 (size=5575) 2024-11-13T03:33:10,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741929_1108 (size=5575) 2024-11-13T03:33:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741929_1108 (size=5575) 2024-11-13T03:33:10,615 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:33:10,617 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 2024-11-13T03:33:10,617 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-13T03:33:10,617 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, journal: Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, size=6.1 K (6243bytes) at 1731468790557Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 so closing down at 1731468790583 (+26 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp at 1731468790608 (+25 ms)3 split writer threads finished at 1731468790609 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468790615 (+6 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000003-wal.1731468790364.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 at 1731468790617 (+2 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731468790617 2024-11-13T03:33:10,619 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790364 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468790364 2024-11-13T03:33:10,620 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 2024-11-13T03:33:10,620 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:10,622 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:10,634 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, exclude list is [], retry=0 2024-11-13T03:33:10,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:10,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:10,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:10,660 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 2024-11-13T03:33:10,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:33:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => acdb77aa047cbf439e7a2da906f4a304, NAME => 'testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:33:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,662 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,663 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName a 2024-11-13T03:33:10,663 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,669 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/40ef013a7ed04cabb5ae331ad19dab93 2024-11-13T03:33:10,669 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,669 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,670 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName b 2024-11-13T03:33:10,670 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,671 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,671 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,672 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName c 2024-11-13T03:33:10,672 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:10,672 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:10,672 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,675 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 2024-11-13T03:33:10,699 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:10,701 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 2024-11-13T03:33:10,701 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acdb77aa047cbf439e7a2da906f4a304 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T03:33:10,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/4174de0d2ba24935b6d300625a17d8c8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468790456/Put/seqid=0 2024-11-13T03:33:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741931_1110 (size=5958) 2024-11-13T03:33:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741931_1110 (size=5958) 2024-11-13T03:33:10,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741931_1110 (size=5958) 2024-11-13T03:33:10,721 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/4174de0d2ba24935b6d300625a17d8c8 2024-11-13T03:33:10,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/195ec41f076a4dcdb8b992be3bb45ed7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731468790462/Put/seqid=0 2024-11-13T03:33:10,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741932_1111 (size=5958) 2024-11-13T03:33:10,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741932_1111 (size=5958) 2024-11-13T03:33:10,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741932_1111 (size=5958) 2024-11-13T03:33:10,745 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/195ec41f076a4dcdb8b992be3bb45ed7 2024-11-13T03:33:10,750 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/4174de0d2ba24935b6d300625a17d8c8 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/b/4174de0d2ba24935b6d300625a17d8c8 2024-11-13T03:33:10,755 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/b/4174de0d2ba24935b6d300625a17d8c8, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T03:33:10,756 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/195ec41f076a4dcdb8b992be3bb45ed7 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/c/195ec41f076a4dcdb8b992be3bb45ed7 2024-11-13T03:33:10,762 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/c/195ec41f076a4dcdb8b992be3bb45ed7, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T03:33:10,762 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for acdb77aa047cbf439e7a2da906f4a304 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T03:33:10,763 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000035 2024-11-13T03:33:10,764 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,764 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,764 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:10,765 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:10,767 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T03:33:10,768 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acdb77aa047cbf439e7a2da906f4a304; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67868892, jitterRate=0.011325299739837646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:10,768 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acdb77aa047cbf439e7a2da906f4a304: Writing region info on filesystem at 1731468790661Initializing all the Stores at 1731468790662 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790662Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790662Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468790662Obtaining lock to block concurrent updates at 1731468790701 (+39 ms)Preparing flush snapshotting stores in acdb77aa047cbf439e7a2da906f4a304 at 1731468790701Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731468790701Flushing stores of testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. at 1731468790701Flushing acdb77aa047cbf439e7a2da906f4a304/b: creating writer at 1731468790701Flushing acdb77aa047cbf439e7a2da906f4a304/b: appending metadata at 1731468790714 (+13 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/b: closing flushed file at 1731468790714Flushing acdb77aa047cbf439e7a2da906f4a304/c: creating writer at 1731468790725 (+11 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/c: appending metadata at 1731468790739 (+14 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/c: closing flushed file at 1731468790739Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fdc2e46: reopening flushed file at 1731468790749 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae10710: reopening flushed file at 1731468790755 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for acdb77aa047cbf439e7a2da906f4a304 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1731468790762 (+7 ms)Cleaning up temporary data from old regions at 1731468790764 (+2 ms)Region opened successfully at 1731468790768 (+4 ms) 2024-11-13T03:33:10,827 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, size=0 (0bytes) 2024-11-13T03:33:10,827 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 might be still open, length is 0 2024-11-13T03:33:10,827 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 2024-11-13T03:33:10,828 WARN [IPC Server handler 1 on default port 33331 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-13T03:33:10,828 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 after 1ms 2024-11-13T03:33:12,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:35236 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39941:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35236 dst: /127.0.0.1:39941 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39941 remote=/127.0.0.1:35236]. Total timeout mills is 60000, 57850 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:12,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:54306 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41693:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54306 dst: /127.0.0.1:41693 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:12,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1148200759_22 at /127.0.0.1:33706 [Receiving block BP-307318758-172.17.0.3-1731468749903:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36333:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33706 dst: /127.0.0.1:36333 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:12,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741930_1112 (size=5567) 2024-11-13T03:33:12,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741930_1112 (size=5567) 2024-11-13T03:33:14,829 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 after 4002ms 2024-11-13T03:33:14,860 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:14,860 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 took 4033ms 2024-11-13T03:33:14,862 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622; continuing. 2024-11-13T03:33:14,862 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 so closing down 2024-11-13T03:33:14,862 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:33:14,863 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731468790622.temp 2024-11-13T03:33:14,888 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp 2024-11-13T03:33:14,888 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:33:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741933_1113 (size=5575) 2024-11-13T03:33:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741933_1113 (size=5575) 2024-11-13T03:33:14,894 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:33:14,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741933_1113 (size=5575) 2024-11-13T03:33:14,895 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 2024-11-13T03:33:14,895 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:33:14,895 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, journal: Splitting hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, size=0 (0bytes) at 1731468790827Finishing writing output for hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 so closing down at 1731468794862 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp at 1731468794888 (+26 ms)3 split writer threads finished at 1731468794888Closed recovered edits writer path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468794894 (+6 ms)Rename recovered edits hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000037-wal.1731468790622.temp to hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 at 1731468794895 (+1 ms)Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622, size=0, length=0, corrupted=false, cancelled=false at 1731468794895 2024-11-13T03:33:14,896 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 to hdfs://localhost:33331/hbase/oldWALs/wal.1731468790622 2024-11-13T03:33:14,897 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 2024-11-13T03:33:14,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:33:14,899 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33331/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281, archiveDir=hdfs://localhost:33331/hbase/oldWALs, maxLogs=32 2024-11-13T03:33:14,912 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468794899, exclude list is [], retry=0 2024-11-13T03:33:14,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39941,DS-7bf710f1-e952-4ef4-b16e-4f9ca33558c4,DISK] 2024-11-13T03:33:14,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36333,DS-5650327c-e752-4068-a9e4-0ba1ca837d86,DISK] 2024-11-13T03:33:14,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41693,DS-889197dd-371a-409e-bdeb-c46bc227082b,DISK] 2024-11-13T03:33:14,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468794899 2024-11-13T03:33:14,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43357:43357),(127.0.0.1/127.0.0.1:37535:37535),(127.0.0.1/127.0.0.1:42365:42365)] 2024-11-13T03:33:14,940 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:33:14,941 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,942 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName a 2024-11-13T03:33:14,942 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:14,947 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/40ef013a7ed04cabb5ae331ad19dab93 2024-11-13T03:33:14,947 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:14,947 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,948 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName b 2024-11-13T03:33:14,948 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:14,953 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/b/4174de0d2ba24935b6d300625a17d8c8 2024-11-13T03:33:14,953 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:14,953 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,954 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdb77aa047cbf439e7a2da906f4a304 columnFamilyName c 2024-11-13T03:33:14,954 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:33:14,960 DEBUG [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/c/195ec41f076a4dcdb8b992be3bb45ed7 2024-11-13T03:33:14,960 INFO [StoreOpener-acdb77aa047cbf439e7a2da906f4a304-1 {}] regionserver.HStore(327): Store=acdb77aa047cbf439e7a2da906f4a304/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:33:14,960 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,962 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:14,963 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 2024-11-13T03:33:14,988 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T03:33:14,991 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 2024-11-13T03:33:14,991 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acdb77aa047cbf439e7a2da906f4a304 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T03:33:15,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/cb0a6bef3710432a85943bb274fd5158 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731468790775/Put/seqid=0 2024-11-13T03:33:15,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741935_1115 (size=5958) 2024-11-13T03:33:15,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741935_1115 (size=5958) 2024-11-13T03:33:15,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741935_1115 (size=5958) 2024-11-13T03:33:15,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/cb0a6bef3710432a85943bb274fd5158 2024-11-13T03:33:15,030 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/29b38d60cf6546c2a10b1d248898d370 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731468790782/Put/seqid=0 2024-11-13T03:33:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741936_1116 (size=5958) 2024-11-13T03:33:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741936_1116 (size=5958) 2024-11-13T03:33:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741936_1116 (size=5958) 2024-11-13T03:33:15,037 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/29b38d60cf6546c2a10b1d248898d370 2024-11-13T03:33:15,056 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/5326c49c3d9a4139821935dda87deea0 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731468790788/Put/seqid=0 2024-11-13T03:33:15,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741937_1117 (size=5958) 2024-11-13T03:33:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741937_1117 (size=5958) 2024-11-13T03:33:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741937_1117 (size=5958) 2024-11-13T03:33:15,062 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/5326c49c3d9a4139821935dda87deea0 2024-11-13T03:33:15,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/a/cb0a6bef3710432a85943bb274fd5158 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/cb0a6bef3710432a85943bb274fd5158 2024-11-13T03:33:15,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/a/cb0a6bef3710432a85943bb274fd5158, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:33:15,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/b/29b38d60cf6546c2a10b1d248898d370 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/b/29b38d60cf6546c2a10b1d248898d370 2024-11-13T03:33:15,078 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/b/29b38d60cf6546c2a10b1d248898d370, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:33:15,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/.tmp/c/5326c49c3d9a4139821935dda87deea0 as hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/c/5326c49c3d9a4139821935dda87deea0 2024-11-13T03:33:15,083 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/c/5326c49c3d9a4139821935dda87deea0, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:33:15,083 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for acdb77aa047cbf439e7a2da906f4a304 in 92ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T03:33:15,084 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/0000000000000000066 2024-11-13T03:33:15,085 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:15,085 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:15,086 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:33:15,087 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acdb77aa047cbf439e7a2da906f4a304 2024-11-13T03:33:15,089 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/hbase/data/default/testReplayEditsWrittenViaHRegion/acdb77aa047cbf439e7a2da906f4a304/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T03:33:15,090 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acdb77aa047cbf439e7a2da906f4a304; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71664113, jitterRate=0.06787849962711334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:33:15,090 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acdb77aa047cbf439e7a2da906f4a304: Writing region info on filesystem at 1731468794940Initializing all the Stores at 1731468794941 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468794941Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468794941Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468794941Obtaining lock to block concurrent updates at 1731468794991 (+50 ms)Preparing flush snapshotting stores in acdb77aa047cbf439e7a2da906f4a304 at 1731468794991Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731468794992 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. at 1731468794992Flushing acdb77aa047cbf439e7a2da906f4a304/a: creating writer at 1731468794992Flushing acdb77aa047cbf439e7a2da906f4a304/a: appending metadata at 1731468795005 (+13 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/a: closing flushed file at 1731468795005Flushing acdb77aa047cbf439e7a2da906f4a304/b: creating writer at 1731468795017 (+12 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/b: appending metadata at 1731468795030 (+13 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/b: closing flushed file at 1731468795030Flushing acdb77aa047cbf439e7a2da906f4a304/c: creating writer at 1731468795040 (+10 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/c: appending metadata at 1731468795055 (+15 ms)Flushing acdb77aa047cbf439e7a2da906f4a304/c: closing flushed file at 1731468795055Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@540509e2: reopening flushed file at 1731468795066 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35c24a6c: reopening flushed file at 1731468795072 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61e537f1: reopening flushed file at 1731468795078 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for acdb77aa047cbf439e7a2da906f4a304 in 92ms, sequenceid=66, compaction requested=false; wal=null at 1731468795083 (+5 ms)Cleaning up temporary data from old regions at 1731468795085 (+2 ms)Region opened successfully at 1731468795090 (+5 ms) 2024-11-13T03:33:15,102 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acdb77aa047cbf439e7a2da906f4a304, disabling compactions & flushes 2024-11-13T03:33:15,102 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:15,102 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:15,102 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. after waiting 0 ms 2024-11-13T03:33:15,103 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:15,104 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468790282.acdb77aa047cbf439e7a2da906f4a304. 2024-11-13T03:33:15,104 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acdb77aa047cbf439e7a2da906f4a304: Waiting for close lock at 1731468795102Disabling compacts and flushes for region at 1731468795102Disabling writes for close at 1731468795103 (+1 ms)Writing region close event to WAL at 1731468795104 (+1 ms)Closed at 1731468795104 2024-11-13T03:33:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741934_1114 (size=175) 2024-11-13T03:33:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741934_1114 (size=175) 2024-11-13T03:33:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741934_1114 (size=175) 2024-11-13T03:33:15,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:33:15,108 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468794899) 2024-11-13T03:33:15,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=448 (was 440) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1382542552_22 at /127.0.0.1:48484 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33331 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1382542552_22 at /127.0.0.1:46950 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1382542552_22 at /127.0.0.1:44984 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33331 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:42547 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42547 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1416 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 157), ProcessCount=11 (was 11), AvailableMemoryMB=2715 (was 2728) 2024-11-13T03:33:15,121 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1416 is superior to 1024 2024-11-13T03:33:15,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T03:33:15,121 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:33:15,121 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:33:15,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:33:15,122 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T03:33:15,122 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=76183466, stopped=false 2024-11-13T03:33:15,123 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8394b182524b,38791,1731468754835 2024-11-13T03:33:15,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T03:33:15,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T03:33:15,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T03:33:15,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:33:15,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:33:15,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:33:15,196 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:33:15,196 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:33:15,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:33:15,196 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,38391,1731468755599' ***** 2024-11-13T03:33:15,196 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T03:33:15,196 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,35657,1731468755703' ***** 2024-11-13T03:33:15,196 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T03:33:15,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:33:15,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:33:15,197 INFO [RS:0;8394b182524b:38391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:33:15,197 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(959): stopping server 8394b182524b,35657,1731468755703 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8394b182524b:35657. 2024-11-13T03:33:15,197 DEBUG [RS:1;8394b182524b:35657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:33:15,197 DEBUG [RS:1;8394b182524b:35657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:33:15,197 INFO [RS:1;8394b182524b:35657 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:33:15,198 INFO [RS:1;8394b182524b:35657 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:33:15,198 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:33:15,198 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(3091): Received CLOSE for adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:33:15,198 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(959): stopping server 8394b182524b,38391,1731468755599 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:33:15,198 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8394b182524b:38391. 2024-11-13T03:33:15,198 DEBUG [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:33:15,198 DEBUG [RS:0;8394b182524b:38391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:33:15,198 DEBUG [RS:0;8394b182524b:38391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adbb549a8a7c08fdb89452cd0ecd22be, disabling compactions & flushes 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:33:15,198 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:33:15,198 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1325): Online Regions={adbb549a8a7c08fdb89452cd0ecd22be=testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be.} 2024-11-13T03:33:15,198 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:33:15,198 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:33:15,198 DEBUG [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1351): Waiting on adbb549a8a7c08fdb89452cd0ecd22be 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. after waiting 0 ms 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:33:15,198 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:33:15,199 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-13T03:33:15,203 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/default/testReplayEditsAfterRegionMovedWithMultiCF/adbb549a8a7c08fdb89452cd0ecd22be/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T03:33:15,203 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:33:15,204 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adbb549a8a7c08fdb89452cd0ecd22be: Waiting for close lock at 1731468795198Running coprocessor pre-close hooks at 1731468795198Disabling compacts and flushes for region at 1731468795198Disabling writes for close at 1731468795198Writing region close event to WAL at 1731468795199 (+1 ms)Running coprocessor post-close hooks at 1731468795203 (+4 ms)Closed at 1731468795203 2024-11-13T03:33:15,204 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be. 2024-11-13T03:33:15,219 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/info/129015360cd7403aaaba70db49a16193 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731468771338.adbb549a8a7c08fdb89452cd0ecd22be./info:regioninfo/1731468774504/Put/seqid=0 2024-11-13T03:33:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741938_1118 (size=8243) 2024-11-13T03:33:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741938_1118 (size=8243) 2024-11-13T03:33:15,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741938_1118 (size=8243) 2024-11-13T03:33:15,226 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/info/129015360cd7403aaaba70db49a16193 2024-11-13T03:33:15,246 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/ns/8ce4924bce1d46c2ba34e1001fbaa797 is 43, key is default/ns:d/1731468758395/Put/seqid=0 2024-11-13T03:33:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741939_1119 (size=5153) 2024-11-13T03:33:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741939_1119 (size=5153) 2024-11-13T03:33:15,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741939_1119 (size=5153) 2024-11-13T03:33:15,256 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/ns/8ce4924bce1d46c2ba34e1001fbaa797 2024-11-13T03:33:15,268 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:33:15,268 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:33:15,276 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/table/2e75e681b1fd456c89d3755f4d0a9f2b is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731468771769/Put/seqid=0 2024-11-13T03:33:15,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741940_1120 (size=5431) 2024-11-13T03:33:15,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741940_1120 (size=5431) 2024-11-13T03:33:15,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741940_1120 (size=5431) 2024-11-13T03:33:15,282 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/table/2e75e681b1fd456c89d3755f4d0a9f2b 2024-11-13T03:33:15,289 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/info/129015360cd7403aaaba70db49a16193 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/info/129015360cd7403aaaba70db49a16193 2024-11-13T03:33:15,295 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/info/129015360cd7403aaaba70db49a16193, entries=18, sequenceid=21, filesize=8.0 K 2024-11-13T03:33:15,296 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/ns/8ce4924bce1d46c2ba34e1001fbaa797 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/ns/8ce4924bce1d46c2ba34e1001fbaa797 2024-11-13T03:33:15,302 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/ns/8ce4924bce1d46c2ba34e1001fbaa797, entries=2, sequenceid=21, filesize=5.0 K 2024-11-13T03:33:15,303 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/.tmp/table/2e75e681b1fd456c89d3755f4d0a9f2b as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/table/2e75e681b1fd456c89d3755f4d0a9f2b 2024-11-13T03:33:15,309 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/table/2e75e681b1fd456c89d3755f4d0a9f2b, entries=2, sequenceid=21, filesize=5.3 K 2024-11-13T03:33:15,310 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=21, compaction requested=false 2024-11-13T03:33:15,315 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T03:33:15,315 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:33:15,315 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:33:15,316 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731468795198Running coprocessor pre-close hooks at 1731468795198Disabling compacts and flushes for region at 1731468795198Disabling writes for close at 1731468795198Obtaining lock to block concurrent updates at 1731468795199 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731468795199Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731468795199Flushing stores of hbase:meta,,1.1588230740 at 1731468795200 (+1 ms)Flushing 1588230740/info: creating writer at 1731468795200Flushing 1588230740/info: appending metadata at 1731468795219 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731468795219Flushing 1588230740/ns: creating writer at 1731468795232 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731468795246 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731468795246Flushing 1588230740/table: creating writer at 1731468795261 (+15 ms)Flushing 1588230740/table: appending metadata at 1731468795275 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731468795275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36dd05b9: reopening flushed file at 1731468795288 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54fe9985: reopening flushed file at 1731468795295 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76f1d16: reopening flushed file at 1731468795302 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=21, compaction requested=false at 1731468795310 (+8 ms)Writing region close event to WAL at 1731468795311 (+1 ms)Running coprocessor post-close hooks at 1731468795315 (+4 ms)Closed at 1731468795315 2024-11-13T03:33:15,316 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T03:33:15,398 INFO [regionserver/8394b182524b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T03:33:15,398 INFO [regionserver/8394b182524b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T03:33:15,398 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,35657,1731468755703; all regions closed. 2024-11-13T03:33:15,398 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,38391,1731468755599; all regions closed. 2024-11-13T03:33:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741833_1009 (size=3621) 2024-11-13T03:33:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741833_1009 (size=3621) 2024-11-13T03:33:15,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741836_1012 (size=9487) 2024-11-13T03:33:15,403 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/WALs/8394b182524b,35657,1731468755703/8394b182524b%2C35657%2C1731468755703.meta.1731468758171.meta not finished, retry = 0 2024-11-13T03:33:15,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741833_1009 (size=3621) 2024-11-13T03:33:15,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741836_1012 (size=9487) 2024-11-13T03:33:15,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741836_1012 (size=9487) 2024-11-13T03:33:15,404 DEBUG [RS:0;8394b182524b:38391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C38391%2C1731468755599:(num 1731468757564) 2024-11-13T03:33:15,405 DEBUG [RS:0;8394b182524b:38391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:33:15,405 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:33:15,405 INFO [RS:0;8394b182524b:38391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38391 2024-11-13T03:33:15,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:33:15,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,38391,1731468755599 2024-11-13T03:33:15,416 INFO [RS:0;8394b182524b:38391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:33:15,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,38391,1731468755599] 2024-11-13T03:33:15,437 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,38391,1731468755599 already deleted, retry=false 2024-11-13T03:33:15,437 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8394b182524b,38391,1731468755599 expired; onlineServers=1 2024-11-13T03:33:15,512 DEBUG [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs 2024-11-13T03:33:15,512 INFO [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C35657%2C1731468755703.meta:.meta(num 1731468758171) 2024-11-13T03:33:15,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741834_1010 (size=175) 2024-11-13T03:33:15,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741834_1010 (size=175) 2024-11-13T03:33:15,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741834_1010 (size=175) 2024-11-13T03:33:15,519 DEBUG [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/oldWALs 2024-11-13T03:33:15,519 INFO [RS:1;8394b182524b:35657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C35657%2C1731468755703:(num 1731468757564) 2024-11-13T03:33:15,519 DEBUG [RS:1;8394b182524b:35657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:33:15,519 INFO [RS:1;8394b182524b:35657 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:33:15,519 INFO [RS:1;8394b182524b:35657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:33:15,520 INFO [RS:1;8394b182524b:35657 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T03:33:15,520 INFO [RS:1;8394b182524b:35657 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:33:15,520 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:33:15,520 INFO [RS:1;8394b182524b:35657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35657 2024-11-13T03:33:15,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,526 INFO [RS:0;8394b182524b:38391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:33:15,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38391-0x101324078460001, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,526 INFO [RS:0;8394b182524b:38391 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,38391,1731468755599; zookeeper connection closed. 2024-11-13T03:33:15,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69cfc54f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69cfc54f 2024-11-13T03:33:15,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,35657,1731468755703 2024-11-13T03:33:15,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:33:15,576 INFO [RS:1;8394b182524b:35657 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:33:15,595 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,35657,1731468755703] 2024-11-13T03:33:15,605 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,35657,1731468755703 already deleted, retry=false 2024-11-13T03:33:15,606 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8394b182524b,35657,1731468755703 expired; onlineServers=0 2024-11-13T03:33:15,606 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8394b182524b,38791,1731468754835' ***** 2024-11-13T03:33:15,606 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T03:33:15,607 INFO [M:0;8394b182524b:38791 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:33:15,607 INFO [M:0;8394b182524b:38791 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:33:15,607 DEBUG [M:0;8394b182524b:38791 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T03:33:15,608 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T03:33:15,608 DEBUG [M:0;8394b182524b:38791 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T03:33:15,608 DEBUG [master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468757126 {}] cleaner.HFileCleaner(306): Exit Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468757126,5,FailOnTimeoutGroup] 2024-11-13T03:33:15,608 DEBUG [master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468757125 {}] cleaner.HFileCleaner(306): Exit Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468757125,5,FailOnTimeoutGroup] 2024-11-13T03:33:15,609 INFO [M:0;8394b182524b:38791 {}] hbase.ChoreService(370): Chore service for: master/8394b182524b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T03:33:15,609 INFO [M:0;8394b182524b:38791 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:33:15,610 DEBUG [M:0;8394b182524b:38791 {}] master.HMaster(1795): Stopping service threads 2024-11-13T03:33:15,610 INFO [M:0;8394b182524b:38791 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T03:33:15,610 INFO [M:0;8394b182524b:38791 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:33:15,611 INFO [M:0;8394b182524b:38791 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T03:33:15,611 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T03:33:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T03:33:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:33:15,616 DEBUG [M:0;8394b182524b:38791 {}] zookeeper.ZKUtil(347): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T03:33:15,616 WARN [M:0;8394b182524b:38791 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T03:33:15,617 INFO [M:0;8394b182524b:38791 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/.lastflushedseqids 2024-11-13T03:33:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741941_1121 (size=138) 2024-11-13T03:33:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741941_1121 (size=138) 2024-11-13T03:33:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741941_1121 (size=138) 2024-11-13T03:33:15,632 INFO [M:0;8394b182524b:38791 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T03:33:15,632 INFO [M:0;8394b182524b:38791 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T03:33:15,632 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:33:15,632 INFO [M:0;8394b182524b:38791 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:33:15,632 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:33:15,632 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:33:15,632 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:33:15,632 INFO [M:0;8394b182524b:38791 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-11-13T03:33:15,646 DEBUG [M:0;8394b182524b:38791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6fb7ce819ef460fb6c0841f94fa5889 is 82, key is hbase:meta,,1/info:regioninfo/1731468758282/Put/seqid=0 2024-11-13T03:33:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741942_1122 (size=5672) 2024-11-13T03:33:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741942_1122 (size=5672) 2024-11-13T03:33:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741942_1122 (size=5672) 2024-11-13T03:33:15,652 INFO [M:0;8394b182524b:38791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6fb7ce819ef460fb6c0841f94fa5889 2024-11-13T03:33:15,671 DEBUG [M:0;8394b182524b:38791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03549f9759eb4a0d8909d26c571a4e4a is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731468771774/Put/seqid=0 2024-11-13T03:33:15,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741943_1123 (size=7754) 2024-11-13T03:33:15,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741943_1123 (size=7754) 2024-11-13T03:33:15,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741943_1123 (size=7754) 2024-11-13T03:33:15,677 INFO [M:0;8394b182524b:38791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03549f9759eb4a0d8909d26c571a4e4a 2024-11-13T03:33:15,681 INFO [M:0;8394b182524b:38791 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03549f9759eb4a0d8909d26c571a4e4a 2024-11-13T03:33:15,695 DEBUG [M:0;8394b182524b:38791 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc56c5a026e0404698e3b8645b12362a is 69, key is 8394b182524b,35657,1731468755703/rs:state/1731468757246/Put/seqid=0 2024-11-13T03:33:15,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,695 INFO [RS:1;8394b182524b:35657 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:33:15,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35657-0x101324078460002, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,695 INFO [RS:1;8394b182524b:35657 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,35657,1731468755703; zookeeper connection closed. 2024-11-13T03:33:15,696 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53802997 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53802997 2024-11-13T03:33:15,696 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T03:33:15,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741944_1124 (size=5440) 2024-11-13T03:33:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741944_1124 (size=5440) 2024-11-13T03:33:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741944_1124 (size=5440) 2024-11-13T03:33:15,701 INFO [M:0;8394b182524b:38791 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc56c5a026e0404698e3b8645b12362a 2024-11-13T03:33:15,705 INFO [M:0;8394b182524b:38791 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc56c5a026e0404698e3b8645b12362a 2024-11-13T03:33:15,706 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6fb7ce819ef460fb6c0841f94fa5889 as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6fb7ce819ef460fb6c0841f94fa5889 2024-11-13T03:33:15,711 INFO [M:0;8394b182524b:38791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6fb7ce819ef460fb6c0841f94fa5889, entries=8, sequenceid=168, filesize=5.5 K 2024-11-13T03:33:15,711 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/03549f9759eb4a0d8909d26c571a4e4a as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/03549f9759eb4a0d8909d26c571a4e4a 2024-11-13T03:33:15,715 INFO [M:0;8394b182524b:38791 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03549f9759eb4a0d8909d26c571a4e4a 2024-11-13T03:33:15,715 INFO [M:0;8394b182524b:38791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/03549f9759eb4a0d8909d26c571a4e4a, entries=17, sequenceid=168, filesize=7.6 K 2024-11-13T03:33:15,716 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc56c5a026e0404698e3b8645b12362a as hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc56c5a026e0404698e3b8645b12362a 2024-11-13T03:33:15,721 INFO [M:0;8394b182524b:38791 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc56c5a026e0404698e3b8645b12362a 2024-11-13T03:33:15,721 INFO [M:0;8394b182524b:38791 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33331/user/jenkins/test-data/aed84da1-0b7d-889c-795a-12216226d5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc56c5a026e0404698e3b8645b12362a, entries=3, sequenceid=168, filesize=5.3 K 2024-11-13T03:33:15,722 INFO [M:0;8394b182524b:38791 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=168, compaction requested=false 2024-11-13T03:33:15,723 INFO [M:0;8394b182524b:38791 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:33:15,723 DEBUG [M:0;8394b182524b:38791 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731468795632Disabling compacts and flushes for region at 1731468795632Disabling writes for close at 1731468795632Obtaining lock to block concurrent updates at 1731468795632Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731468795632Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1731468795632Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731468795633 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731468795633Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731468795646 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731468795646Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731468795656 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731468795670 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731468795670Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731468795681 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731468795694 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731468795694Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9512f3: reopening flushed file at 1731468795705 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cc65ddf: reopening flushed file at 1731468795711 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44fb6daa: reopening flushed file at 1731468795716 (+5 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=168, compaction requested=false at 1731468795722 (+6 ms)Writing region close event to WAL at 1731468795723 (+1 ms)Closed at 1731468795723 2024-11-13T03:33:15,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741830_1006 (size=83594) 2024-11-13T03:33:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36333 is added to blk_1073741830_1006 (size=83594) 2024-11-13T03:33:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41693 is added to blk_1073741830_1006 (size=83594) 2024-11-13T03:33:15,726 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:33:15,726 INFO [M:0;8394b182524b:38791 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T03:33:15,727 INFO [M:0;8394b182524b:38791 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38791 2024-11-13T03:33:15,727 INFO [M:0;8394b182524b:38791 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:33:15,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,832 INFO [M:0;8394b182524b:38791 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:33:15,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38791-0x101324078460000, quorum=127.0.0.1:56173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:33:15,843 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468775139 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468775139 (inode 16655) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468774793/wal.1731468775139 (inode 16655) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:33:15,844 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468766799/wal.1731468766926 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:15,844 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468790281/wal.1731468790622 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:15,847 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468784659 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468784659 (inode 16678) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468775315/wal.1731468784659 (inode 16678) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:33:15,847 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468759926/wal.1731468760023 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:15,849 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468785454 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468785454 (inode 16704) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468784909/wal.1731468785454 (inode 16704) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:33:15,850 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468785699/wal.1731468785797 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:33:15,851 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal.1731468759734 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal.1731468759734 (inode 16485) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468759355/wal.1731468759734 (inode 16485) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:33:15,853 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal.1731468759032 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal.1731468759032 (inode 16462) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468758586/wal.1731468759032 (inode 16462) Holder DFSClient_NONMAPREDUCE_1148200759_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:33:15,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:33:15,859 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:33:15,859 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:33:15,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:33:15,860 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,STOPPED} 2024-11-13T03:33:15,862 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:33:15,862 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:33:15,862 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-307318758-172.17.0.3-1731468749903 (Datanode Uuid 69c4d9cd-316e-41f2-b74b-20110b5622c2) service to localhost/127.0.0.1:33331 2024-11-13T03:33:15,862 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:33:15,864 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data5/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,864 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data6/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,864 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:33:15,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:33:15,866 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:33:15,866 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:33:15,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:33:15,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,STOPPED} 2024-11-13T03:33:15,868 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:33:15,868 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:33:15,868 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:33:15,868 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-307318758-172.17.0.3-1731468749903 (Datanode Uuid f905237a-6464-4985-a710-f3ea5d90bb18) service to localhost/127.0.0.1:33331 2024-11-13T03:33:15,869 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data3/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,869 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data4/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,869 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:33:15,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:33:15,871 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:33:15,871 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:33:15,872 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:33:15,872 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,STOPPED} 2024-11-13T03:33:15,873 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:33:15,873 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:33:15,873 WARN [BP-307318758-172.17.0.3-1731468749903 heartbeating to localhost/127.0.0.1:33331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-307318758-172.17.0.3-1731468749903 (Datanode Uuid 5d84471c-1613-4e35-ac39-357dec741b01) service to localhost/127.0.0.1:33331 2024-11-13T03:33:15,873 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:33:15,874 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data1/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,874 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/cluster_e8140803-49ac-e4fd-f6ef-f9f846f95351/data/data2/current/BP-307318758-172.17.0.3-1731468749903 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:33:15,874 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:33:15,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:33:15,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:33:15,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:33:15,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:33:15,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/22a52be4-af30-78cb-2d8d-584f2b767131/hadoop.log.dir/,STOPPED} 2024-11-13T03:33:15,892 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T03:33:15,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down