2024-11-21 12:30:11,003 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-21 12:30:11,017 main DEBUG Took 0.011266 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 12:30:11,017 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 12:30:11,018 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 12:30:11,019 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 12:30:11,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,028 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 12:30:11,040 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,042 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,043 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,044 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,045 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,045 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,046 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,046 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,047 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,048 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,048 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,049 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,050 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 12:30:11,051 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,051 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 12:30:11,052 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 12:30:11,054 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 12:30:11,056 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 12:30:11,056 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 12:30:11,057 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 12:30:11,058 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 12:30:11,067 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 12:30:11,070 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 12:30:11,072 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 12:30:11,072 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 12:30:11,073 main DEBUG createAppenders(={Console}) 2024-11-21 12:30:11,073 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-21 12:30:11,074 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-21 12:30:11,074 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-21 12:30:11,074 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 12:30:11,075 main DEBUG OutputStream closed 2024-11-21 12:30:11,075 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 12:30:11,075 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 12:30:11,075 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-21 12:30:11,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 12:30:11,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 12:30:11,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 12:30:11,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 12:30:11,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 12:30:11,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 12:30:11,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 12:30:11,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 12:30:11,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 12:30:11,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 12:30:11,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 12:30:11,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 12:30:11,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 12:30:11,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 12:30:11,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 12:30:11,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 12:30:11,159 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 12:30:11,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 12:30:11,163 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 12:30:11,163 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-21 12:30:11,164 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 12:30:11,164 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-21T12:30:11,411 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2 2024-11-21 12:30:11,414 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 12:30:11,414 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T12:30:11,424 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-21T12:30:11,431 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-21T12:30:11,453 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T12:30:11,502 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T12:30:11,502 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T12:30:11,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T12:30:11,531 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd, deleteOnExit=true 2024-11-21T12:30:11,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T12:30:11,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/test.cache.data in system properties and HBase conf 2024-11-21T12:30:11,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T12:30:11,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir in system properties and HBase conf 2024-11-21T12:30:11,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T12:30:11,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T12:30:11,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T12:30:11,645 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T12:30:11,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T12:30:11,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T12:30:11,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T12:30:11,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T12:30:11,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T12:30:11,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T12:30:11,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T12:30:11,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T12:30:11,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T12:30:11,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T12:30:11,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/nfs.dump.dir in system properties and HBase conf 2024-11-21T12:30:11,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/java.io.tmpdir in system properties and HBase conf 2024-11-21T12:30:11,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T12:30:11,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T12:30:11,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T12:30:12,705 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T12:30:12,793 INFO [Time-limited test {}] log.Log(170): Logging initialized @2584ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T12:30:12,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T12:30:12,971 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T12:30:12,993 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T12:30:12,993 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T12:30:12,994 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T12:30:13,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T12:30:13,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,AVAILABLE} 2024-11-21T12:30:13,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T12:30:13,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/java.io.tmpdir/jetty-localhost-45847-hadoop-hdfs-3_4_1-tests_jar-_-any-9450031576104771711/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T12:30:13,232 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:45847} 2024-11-21T12:30:13,232 INFO [Time-limited test {}] server.Server(415): Started @3023ms 2024-11-21T12:30:13,660 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T12:30:13,670 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T12:30:13,671 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T12:30:13,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T12:30:13,672 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T12:30:13,673 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,AVAILABLE} 2024-11-21T12:30:13,673 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T12:30:13,796 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/java.io.tmpdir/jetty-localhost-43377-hadoop-hdfs-3_4_1-tests_jar-_-any-9938360641319890486/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:13,797 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:43377} 2024-11-21T12:30:13,797 INFO [Time-limited test {}] server.Server(415): Started @3588ms 2024-11-21T12:30:13,866 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T12:30:14,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T12:30:14,016 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T12:30:14,018 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T12:30:14,018 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T12:30:14,019 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T12:30:14,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,AVAILABLE} 2024-11-21T12:30:14,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T12:30:14,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/java.io.tmpdir/jetty-localhost-44239-hadoop-hdfs-3_4_1-tests_jar-_-any-930820538382177369/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:14,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:44239} 2024-11-21T12:30:14,145 INFO [Time-limited test {}] server.Server(415): Started @3935ms 2024-11-21T12:30:14,148 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T12:30:14,192 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T12:30:14,196 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T12:30:14,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T12:30:14,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T12:30:14,199 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T12:30:14,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,AVAILABLE} 2024-11-21T12:30:14,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T12:30:14,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/java.io.tmpdir/jetty-localhost-42339-hadoop-hdfs-3_4_1-tests_jar-_-any-6867384497935344368/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:14,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:42339} 2024-11-21T12:30:14,330 INFO [Time-limited test {}] server.Server(415): Started @4121ms 2024-11-21T12:30:14,333 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T12:30:14,347 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data3/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,347 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data4/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,347 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data1/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,347 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data2/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,399 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T12:30:14,400 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T12:30:14,462 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data5/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,462 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data6/current/BP-1168079295-172.17.0.2-1732192212418/current, will proceed with Du for space computation calculation, 2024-11-21T12:30:14,497 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x38a90ce778d5c1bf with lease ID 0x3c40f2ab47fc7828: Processing first storage report for DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959 from datanode DatanodeRegistration(127.0.0.1:37507, datanodeUuid=f7762d60-0d12-48a7-a3e4-9ddf77859730, infoPort=33699, infoSecurePort=0, ipcPort=37147, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,498 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x38a90ce778d5c1bf with lease ID 0x3c40f2ab47fc7828: from storage DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959 node DatanodeRegistration(127.0.0.1:37507, datanodeUuid=f7762d60-0d12-48a7-a3e4-9ddf77859730, infoPort=33699, infoSecurePort=0, ipcPort=37147, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,499 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad9510d839d4766c with lease ID 0x3c40f2ab47fc7829: Processing first storage report for DS-994861ff-3fbd-4017-8303-85affa12688c from datanode DatanodeRegistration(127.0.0.1:34335, datanodeUuid=ae61627a-96b8-446d-bd16-0c59bc3292ba, infoPort=33707, infoSecurePort=0, ipcPort=37441, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,499 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad9510d839d4766c with lease ID 0x3c40f2ab47fc7829: from storage DS-994861ff-3fbd-4017-8303-85affa12688c node DatanodeRegistration(127.0.0.1:34335, datanodeUuid=ae61627a-96b8-446d-bd16-0c59bc3292ba, infoPort=33707, infoSecurePort=0, ipcPort=37441, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x38a90ce778d5c1bf with lease ID 0x3c40f2ab47fc7828: Processing first storage report for DS-e3505164-1e48-4b1a-b64c-02a7505e9c27 from datanode DatanodeRegistration(127.0.0.1:37507, datanodeUuid=f7762d60-0d12-48a7-a3e4-9ddf77859730, infoPort=33699, infoSecurePort=0, ipcPort=37147, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x38a90ce778d5c1bf with lease ID 0x3c40f2ab47fc7828: from storage DS-e3505164-1e48-4b1a-b64c-02a7505e9c27 node DatanodeRegistration(127.0.0.1:37507, datanodeUuid=f7762d60-0d12-48a7-a3e4-9ddf77859730, infoPort=33699, infoSecurePort=0, ipcPort=37147, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad9510d839d4766c with lease ID 0x3c40f2ab47fc7829: Processing first storage report for DS-00feffba-aa07-49c5-95ee-5082f75d2917 from datanode DatanodeRegistration(127.0.0.1:34335, datanodeUuid=ae61627a-96b8-446d-bd16-0c59bc3292ba, infoPort=33707, infoSecurePort=0, ipcPort=37441, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad9510d839d4766c with lease ID 0x3c40f2ab47fc7829: from storage DS-00feffba-aa07-49c5-95ee-5082f75d2917 node DatanodeRegistration(127.0.0.1:34335, datanodeUuid=ae61627a-96b8-446d-bd16-0c59bc3292ba, infoPort=33707, infoSecurePort=0, ipcPort=37441, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,501 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T12:30:14,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53f931ac8a0a36d0 with lease ID 0x3c40f2ab47fc782a: Processing first storage report for DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28 from datanode DatanodeRegistration(127.0.0.1:39871, datanodeUuid=411fc6f6-12a3-407f-805e-27aee14c66de, infoPort=44529, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53f931ac8a0a36d0 with lease ID 0x3c40f2ab47fc782a: from storage DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28 node DatanodeRegistration(127.0.0.1:39871, datanodeUuid=411fc6f6-12a3-407f-805e-27aee14c66de, infoPort=44529, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,513 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x53f931ac8a0a36d0 with lease ID 0x3c40f2ab47fc782a: Processing first storage report for DS-9d781e7d-8d82-43f4-8073-708a8264ad34 from datanode DatanodeRegistration(127.0.0.1:39871, datanodeUuid=411fc6f6-12a3-407f-805e-27aee14c66de, infoPort=44529, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418) 2024-11-21T12:30:14,514 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x53f931ac8a0a36d0 with lease ID 0x3c40f2ab47fc782a: from storage DS-9d781e7d-8d82-43f4-8073-708a8264ad34 node DatanodeRegistration(127.0.0.1:39871, datanodeUuid=411fc6f6-12a3-407f-805e-27aee14c66de, infoPort=44529, infoSecurePort=0, ipcPort=35067, storageInfo=lv=-57;cid=testClusterID;nsid=1314684665;c=1732192212418), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T12:30:14,753 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2 2024-11-21T12:30:14,846 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/zookeeper_0, clientPort=55541, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T12:30:14,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55541 2024-11-21T12:30:14,905 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:14,909 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741825_1001 (size=7) 2024-11-21T12:30:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741825_1001 (size=7) 2024-11-21T12:30:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741825_1001 (size=7) 2024-11-21T12:30:15,628 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 with version=8 2024-11-21T12:30:15,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/hbase-staging 2024-11-21T12:30:15,966 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9bb5b7e9dd2c:0 server-side Connection retries=45 2024-11-21T12:30:15,978 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:15,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:15,984 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T12:30:15,984 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:15,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T12:30:16,143 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T12:30:16,212 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T12:30:16,226 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T12:30:16,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T12:30:16,266 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17686 (auto-detected) 2024-11-21T12:30:16,268 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T12:30:16,291 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32899 2024-11-21T12:30:16,314 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32899 connecting to ZooKeeper ensemble=127.0.0.1:55541 2024-11-21T12:30:16,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328990x0, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T12:30:16,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32899-0x1006251a9c40000 connected 2024-11-21T12:30:16,386 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:16,407 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6, hbase.cluster.distributed=false 2024-11-21T12:30:16,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T12:30:16,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-21T12:30:16,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32899 2024-11-21T12:30:16,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32899 2024-11-21T12:30:16,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-21T12:30:16,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-21T12:30:16,577 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9bb5b7e9dd2c:0 server-side Connection retries=45 2024-11-21T12:30:16,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,579 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T12:30:16,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T12:30:16,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T12:30:16,585 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T12:30:16,586 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41613 2024-11-21T12:30:16,588 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41613 connecting to ZooKeeper ensemble=127.0.0.1:55541 2024-11-21T12:30:16,590 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,596 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416130x0, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T12:30:16,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41613-0x1006251a9c40001 connected 2024-11-21T12:30:16,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:16,615 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T12:30:16,626 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T12:30:16,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T12:30:16,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T12:30:16,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41613 2024-11-21T12:30:16,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41613 2024-11-21T12:30:16,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41613 2024-11-21T12:30:16,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41613 2024-11-21T12:30:16,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41613 2024-11-21T12:30:16,659 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9bb5b7e9dd2c:0 server-side Connection retries=45 2024-11-21T12:30:16,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,661 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T12:30:16,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T12:30:16,661 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T12:30:16,662 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T12:30:16,663 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36317 2024-11-21T12:30:16,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36317 connecting to ZooKeeper ensemble=127.0.0.1:55541 2024-11-21T12:30:16,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,670 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363170x0, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T12:30:16,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:16,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36317-0x1006251a9c40002 connected 2024-11-21T12:30:16,678 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T12:30:16,680 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T12:30:16,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T12:30:16,683 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T12:30:16,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36317 2024-11-21T12:30:16,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36317 2024-11-21T12:30:16,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36317 2024-11-21T12:30:16,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36317 2024-11-21T12:30:16,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36317 2024-11-21T12:30:16,705 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9bb5b7e9dd2c:0 server-side Connection retries=45 2024-11-21T12:30:16,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,705 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T12:30:16,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T12:30:16,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T12:30:16,706 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T12:30:16,706 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T12:30:16,708 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44681 2024-11-21T12:30:16,709 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44681 connecting to ZooKeeper ensemble=127.0.0.1:55541 2024-11-21T12:30:16,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,714 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446810x0, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T12:30:16,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446810x0, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:16,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44681-0x1006251a9c40003 connected 2024-11-21T12:30:16,723 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T12:30:16,724 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T12:30:16,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T12:30:16,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T12:30:16,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44681 2024-11-21T12:30:16,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44681 2024-11-21T12:30:16,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44681 2024-11-21T12:30:16,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44681 2024-11-21T12:30:16,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44681 2024-11-21T12:30:16,753 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9bb5b7e9dd2c:32899 2024-11-21T12:30:16,755 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:16,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,767 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:16,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T12:30:16,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T12:30:16,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T12:30:16,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,792 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T12:30:16,793 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9bb5b7e9dd2c,32899,1732192215752 from backup master directory 2024-11-21T12:30:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T12:30:16,798 WARN [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T12:30:16,798 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:16,801 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T12:30:16,803 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T12:30:16,878 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/hbase.id] with ID: 0ec7614b-bbbd-45b9-88cc-598d55c1d096 2024-11-21T12:30:16,878 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/.tmp/hbase.id 2024-11-21T12:30:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741826_1002 (size=42) 2024-11-21T12:30:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741826_1002 (size=42) 2024-11-21T12:30:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741826_1002 (size=42) 2024-11-21T12:30:16,895 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/.tmp/hbase.id]:[hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/hbase.id] 2024-11-21T12:30:16,948 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T12:30:16,953 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T12:30:16,976 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-21T12:30:16,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:16,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741827_1003 (size=196) 2024-11-21T12:30:17,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741827_1003 (size=196) 2024-11-21T12:30:17,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741827_1003 (size=196) 2024-11-21T12:30:17,026 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T12:30:17,028 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T12:30:17,035 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:17,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741828_1004 (size=1189) 2024-11-21T12:30:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741828_1004 (size=1189) 2024-11-21T12:30:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741828_1004 (size=1189) 2024-11-21T12:30:17,096 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store 2024-11-21T12:30:17,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741829_1005 (size=34) 2024-11-21T12:30:17,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741829_1005 (size=34) 2024-11-21T12:30:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741829_1005 (size=34) 2024-11-21T12:30:17,129 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T12:30:17,134 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:17,136 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T12:30:17,136 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:17,137 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:17,139 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T12:30:17,139 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:17,139 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:17,141 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732192217136Disabling compacts and flushes for region at 1732192217136Disabling writes for close at 1732192217139 (+3 ms)Writing region close event to WAL at 1732192217139Closed at 1732192217139 2024-11-21T12:30:17,144 WARN [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/.initializing 2024-11-21T12:30:17,144 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/WALs/9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:17,155 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:17,177 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C32899%2C1732192215752, suffix=, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/WALs/9bb5b7e9dd2c,32899,1732192215752, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/oldWALs, maxLogs=10 2024-11-21T12:30:17,226 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/WALs/9bb5b7e9dd2c,32899,1732192215752/9bb5b7e9dd2c%2C32899%2C1732192215752.1732192217184, exclude list is [], retry=0 2024-11-21T12:30:17,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:17,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:17,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:17,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:17,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T12:30:17,308 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-21T12:30:17,388 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/WALs/9bb5b7e9dd2c,32899,1732192215752/9bb5b7e9dd2c%2C32899%2C1732192215752.1732192217184 2024-11-21T12:30:17,389 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:17,390 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:17,390 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:17,394 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,394 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T12:30:17,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:17,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:17,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T12:30:17,489 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:17,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:17,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T12:30:17,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:17,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:17,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T12:30:17,499 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:17,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:17,501 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,506 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,508 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,515 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,516 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,521 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T12:30:17,525 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T12:30:17,530 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:17,532 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64924493, jitterRate=-0.0325496643781662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T12:30:17,541 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732192217408Initializing all the Stores at 1732192217411 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192217412 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192217413 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192217413Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192217413Cleaning up temporary data from old regions at 1732192217516 (+103 ms)Region opened successfully at 1732192217540 (+24 ms) 2024-11-21T12:30:17,542 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T12:30:17,586 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7918918c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9bb5b7e9dd2c/172.17.0.2:0 2024-11-21T12:30:17,626 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T12:30:17,641 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T12:30:17,641 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T12:30:17,645 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T12:30:17,646 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T12:30:17,652 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T12:30:17,652 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T12:30:17,682 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T12:30:17,695 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T12:30:17,698 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T12:30:17,702 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T12:30:17,705 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T12:30:17,707 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T12:30:17,711 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T12:30:17,715 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T12:30:17,717 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T12:30:17,719 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T12:30:17,721 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T12:30:17,742 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T12:30:17,744 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:17,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,753 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9bb5b7e9dd2c,32899,1732192215752, sessionid=0x1006251a9c40000, setting cluster-up flag (Was=false) 2024-11-21T12:30:17,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,774 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T12:30:17,777 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:17,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:17,790 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T12:30:17,792 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:17,799 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T12:30:17,836 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(746): ClusterId : 0ec7614b-bbbd-45b9-88cc-598d55c1d096 2024-11-21T12:30:17,836 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(746): ClusterId : 0ec7614b-bbbd-45b9-88cc-598d55c1d096 2024-11-21T12:30:17,836 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(746): ClusterId : 0ec7614b-bbbd-45b9-88cc-598d55c1d096 2024-11-21T12:30:17,839 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T12:30:17,839 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T12:30:17,839 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T12:30:17,847 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T12:30:17,847 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T12:30:17,847 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T12:30:17,847 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T12:30:17,849 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T12:30:17,849 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T12:30:17,853 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T12:30:17,853 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T12:30:17,853 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T12:30:17,853 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79350a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9bb5b7e9dd2c/172.17.0.2:0 2024-11-21T12:30:17,853 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42eacee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9bb5b7e9dd2c/172.17.0.2:0 2024-11-21T12:30:17,853 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdccdfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9bb5b7e9dd2c/172.17.0.2:0 2024-11-21T12:30:17,875 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9bb5b7e9dd2c:44681 2024-11-21T12:30:17,879 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9bb5b7e9dd2c:36317 2024-11-21T12:30:17,879 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9bb5b7e9dd2c:41613 2024-11-21T12:30:17,879 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T12:30:17,879 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T12:30:17,879 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T12:30:17,879 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T12:30:17,880 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T12:30:17,880 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T12:30:17,880 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T12:30:17,880 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T12:30:17,880 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T12:30:17,883 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=44681, startcode=1732192216704 2024-11-21T12:30:17,883 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=36317, startcode=1732192216659 2024-11-21T12:30:17,883 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=41613, startcode=1732192216530 2024-11-21T12:30:17,899 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T12:30:17,899 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T12:30:17,899 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T12:30:17,901 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T12:30:17,916 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T12:30:17,925 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T12:30:17,932 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9bb5b7e9dd2c,32899,1732192215752 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T12:30:17,948 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50073, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T12:30:17,948 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T12:30:17,948 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52325, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T12:30:17,948 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9bb5b7e9dd2c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T12:30:17,948 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9bb5b7e9dd2c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T12:30:17,949 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9bb5b7e9dd2c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T12:30:17,949 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9bb5b7e9dd2c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T12:30:17,949 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9bb5b7e9dd2c:0, corePoolSize=10, maxPoolSize=10 2024-11-21T12:30:17,949 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:17,949 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9bb5b7e9dd2c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T12:30:17,950 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:17,954 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732192247954 2024-11-21T12:30:17,957 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T12:30:17,956 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T12:30:17,958 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T12:30:17,960 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T12:30:17,960 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T12:30:17,962 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T12:30:17,963 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T12:30:17,963 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T12:30:17,963 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T12:30:17,964 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:17,967 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T12:30:17,968 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T12:30:17,970 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T12:30:17,971 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T12:30:17,972 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:17,972 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T12:30:17,972 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T12:30:17,981 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T12:30:17,981 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T12:30:17,986 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.large.0-1732192217985,5,FailOnTimeoutGroup] 2024-11-21T12:30:17,987 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.small.0-1732192217986,5,FailOnTimeoutGroup] 2024-11-21T12:30:17,987 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:17,988 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T12:30:17,989 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:17,990 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:17,993 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T12:30:17,993 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T12:30:17,993 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T12:30:17,993 WARN [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T12:30:17,993 WARN [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T12:30:17,993 WARN [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T12:30:18,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741831_1007 (size=1321) 2024-11-21T12:30:18,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741831_1007 (size=1321) 2024-11-21T12:30:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741831_1007 (size=1321) 2024-11-21T12:30:18,024 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T12:30:18,024 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 2024-11-21T12:30:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741832_1008 (size=32) 2024-11-21T12:30:18,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741832_1008 (size=32) 2024-11-21T12:30:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741832_1008 (size=32) 2024-11-21T12:30:18,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:18,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T12:30:18,068 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T12:30:18,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T12:30:18,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T12:30:18,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,076 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T12:30:18,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T12:30:18,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T12:30:18,084 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T12:30:18,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T12:30:18,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:18,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:18,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T12:30:18,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T12:30:18,092 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T12:30:18,095 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=41613, startcode=1732192216530 2024-11-21T12:30:18,095 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=44681, startcode=1732192216704 2024-11-21T12:30:18,095 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(2659): reportForDuty to master=9bb5b7e9dd2c,32899,1732192215752 with port=36317, startcode=1732192216659 2024-11-21T12:30:18,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T12:30:18,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,101 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:18,101 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(517): Registering regionserver=9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,102 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71326158, jitterRate=0.06284257769584656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T12:30:18,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732192218062Initializing all the Stores at 1732192218064 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218064Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218064Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192218065 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218065Cleaning up temporary data from old regions at 1732192218091 (+26 ms)Region opened successfully at 1732192218106 (+15 ms) 2024-11-21T12:30:18,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T12:30:18,107 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T12:30:18,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T12:30:18,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T12:30:18,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T12:30:18,110 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T12:30:18,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732192218107Disabling compacts and flushes for region at 1732192218107Disabling writes for close at 1732192218108 (+1 ms)Writing region close event to WAL at 1732192218109 (+1 ms)Closed at 1732192218110 (+1 ms) 2024-11-21T12:30:18,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(517): Registering regionserver=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,114 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 2024-11-21T12:30:18,114 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41319 2024-11-21T12:30:18,114 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T12:30:18,117 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T12:30:18,117 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T12:30:18,119 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,119 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 2024-11-21T12:30:18,119 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41319 2024-11-21T12:30:18,119 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T12:30:18,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T12:30:18,120 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32899 {}] master.ServerManager(517): Registering regionserver=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,121 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] zookeeper.ZKUtil(111): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,121 WARN [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T12:30:18,121 INFO [RS:2;9bb5b7e9dd2c:44681 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:18,122 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T12:30:18,128 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] zookeeper.ZKUtil(111): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,128 WARN [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T12:30:18,128 INFO [RS:1;9bb5b7e9dd2c:36317 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:18,128 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9bb5b7e9dd2c,44681,1732192216704] 2024-11-21T12:30:18,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9bb5b7e9dd2c,36317,1732192216659] 2024-11-21T12:30:18,129 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 2024-11-21T12:30:18,129 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41319 2024-11-21T12:30:18,129 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T12:30:18,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T12:30:18,133 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] zookeeper.ZKUtil(111): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,133 WARN [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T12:30:18,133 INFO [RS:0;9bb5b7e9dd2c:41613 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:18,133 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,137 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9bb5b7e9dd2c,41613,1732192216530] 2024-11-21T12:30:18,141 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T12:30:18,149 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T12:30:18,175 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T12:30:18,175 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T12:30:18,175 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T12:30:18,190 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T12:30:18,190 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T12:30:18,191 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T12:30:18,196 INFO [RS:0;9bb5b7e9dd2c:41613 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T12:30:18,196 INFO [RS:1;9bb5b7e9dd2c:36317 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T12:30:18,196 INFO [RS:2;9bb5b7e9dd2c:44681 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T12:30:18,197 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,197 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,197 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,205 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T12:30:18,205 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T12:30:18,205 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T12:30:18,213 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T12:30:18,213 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T12:30:18,213 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T12:30:18,215 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,215 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,215 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,215 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,215 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,215 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T12:30:18,216 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,216 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T12:30:18,216 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,217 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,217 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,217 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,218 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,218 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,218 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,218 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9bb5b7e9dd2c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T12:30:18,218 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,218 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9bb5b7e9dd2c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T12:30:18,229 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,230 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,230 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,230 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,230 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,230 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,36317,1732192216659-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T12:30:18,231 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,231 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,232 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,232 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,232 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,232 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,41613,1732192216530-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,234 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,44681,1732192216704-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T12:30:18,256 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T12:30:18,259 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,36317,1732192216659-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,259 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T12:30:18,260 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,41613,1732192216530-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,260 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,260 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,260 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.Replication(171): 9bb5b7e9dd2c,41613,1732192216530 started 2024-11-21T12:30:18,260 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.Replication(171): 9bb5b7e9dd2c,36317,1732192216659 started 2024-11-21T12:30:18,268 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T12:30:18,269 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,44681,1732192216704-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,269 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,270 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.Replication(171): 9bb5b7e9dd2c,44681,1732192216704 started 2024-11-21T12:30:18,289 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,289 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,290 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1482): Serving as 9bb5b7e9dd2c,36317,1732192216659, RpcServer on 9bb5b7e9dd2c/172.17.0.2:36317, sessionid=0x1006251a9c40002 2024-11-21T12:30:18,290 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1482): Serving as 9bb5b7e9dd2c,41613,1732192216530, RpcServer on 9bb5b7e9dd2c/172.17.0.2:41613, sessionid=0x1006251a9c40001 2024-11-21T12:30:18,291 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T12:30:18,291 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T12:30:18,291 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,291 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,291 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,36317,1732192216659' 2024-11-21T12:30:18,291 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,41613,1732192216530' 2024-11-21T12:30:18,291 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T12:30:18,291 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T12:30:18,293 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T12:30:18,293 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T12:30:18,294 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T12:30:18,294 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T12:30:18,294 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T12:30:18,294 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T12:30:18,294 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:18,294 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,294 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,36317,1732192216659' 2024-11-21T12:30:18,294 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,41613,1732192216530' 2024-11-21T12:30:18,294 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T12:30:18,294 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T12:30:18,295 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T12:30:18,295 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T12:30:18,296 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T12:30:18,296 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T12:30:18,296 INFO [RS:1;9bb5b7e9dd2c:36317 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T12:30:18,296 INFO [RS:0;9bb5b7e9dd2c:41613 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T12:30:18,296 INFO [RS:1;9bb5b7e9dd2c:36317 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T12:30:18,296 INFO [RS:0;9bb5b7e9dd2c:41613 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T12:30:18,300 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:18,300 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1482): Serving as 9bb5b7e9dd2c,44681,1732192216704, RpcServer on 9bb5b7e9dd2c/172.17.0.2:44681, sessionid=0x1006251a9c40003 2024-11-21T12:30:18,300 WARN [9bb5b7e9dd2c:32899 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T12:30:18,301 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T12:30:18,301 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,301 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,44681,1732192216704' 2024-11-21T12:30:18,301 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T12:30:18,302 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T12:30:18,303 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T12:30:18,303 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T12:30:18,303 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:18,303 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9bb5b7e9dd2c,44681,1732192216704' 2024-11-21T12:30:18,303 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T12:30:18,304 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T12:30:18,305 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T12:30:18,305 INFO [RS:2;9bb5b7e9dd2c:44681 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T12:30:18,305 INFO [RS:2;9bb5b7e9dd2c:44681 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T12:30:18,405 INFO [RS:0;9bb5b7e9dd2c:41613 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:18,405 INFO [RS:1;9bb5b7e9dd2c:36317 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:18,405 INFO [RS:2;9bb5b7e9dd2c:44681 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:18,409 INFO [RS:0;9bb5b7e9dd2c:41613 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C41613%2C1732192216530, suffix=, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs, maxLogs=32 2024-11-21T12:30:18,409 INFO [RS:2;9bb5b7e9dd2c:44681 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C44681%2C1732192216704, suffix=, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,44681,1732192216704, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs, maxLogs=32 2024-11-21T12:30:18,409 INFO [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C36317%2C1732192216659, suffix=, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs, maxLogs=32 2024-11-21T12:30:18,437 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659/9bb5b7e9dd2c%2C36317%2C1732192216659.1732192218415, exclude list is [], retry=0 2024-11-21T12:30:18,437 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,44681,1732192216704/9bb5b7e9dd2c%2C44681%2C1732192216704.1732192218415, exclude list is [], retry=0 2024-11-21T12:30:18,437 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, exclude list is [], retry=0 2024-11-21T12:30:18,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:18,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:18,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:18,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:18,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:18,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:18,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:18,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:18,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:18,511 INFO [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659/9bb5b7e9dd2c%2C36317%2C1732192216659.1732192218415 2024-11-21T12:30:18,512 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:18,518 INFO [RS:0;9bb5b7e9dd2c:41613 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 2024-11-21T12:30:18,518 INFO [RS:2;9bb5b7e9dd2c:44681 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,44681,1732192216704/9bb5b7e9dd2c%2C44681%2C1732192216704.1732192218415 2024-11-21T12:30:18,519 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:18,521 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:18,553 DEBUG [9bb5b7e9dd2c:32899 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-21T12:30:18,563 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(204): Hosts are {9bb5b7e9dd2c=0} racks are {/default-rack=0} 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T12:30:18,572 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T12:30:18,572 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T12:30:18,572 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T12:30:18,572 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T12:30:18,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,590 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9bb5b7e9dd2c,41613,1732192216530, state=OPENING 2024-11-21T12:30:18,599 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T12:30:18,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:18,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:18,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:18,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:18,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,607 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T12:30:18,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,41613,1732192216530}] 2024-11-21T12:30:18,792 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T12:30:18,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50891, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T12:30:18,809 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T12:30:18,810 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:18,811 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T12:30:18,814 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C41613%2C1732192216530.meta, suffix=.meta, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs, maxLogs=32 2024-11-21T12:30:18,832 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, exclude list is [], retry=0 2024-11-21T12:30:18,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:18,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:18,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:18,874 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta 2024-11-21T12:30:18,875 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:18,875 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:18,878 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T12:30:18,881 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T12:30:18,888 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T12:30:18,893 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T12:30:18,894 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:18,894 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T12:30:18,894 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T12:30:18,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T12:30:18,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T12:30:18,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T12:30:18,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T12:30:18,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T12:30:18,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T12:30:18,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T12:30:18,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T12:30:18,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:18,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:18,912 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T12:30:18,914 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:18,928 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:18,931 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T12:30:18,932 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T12:30:18,933 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T12:30:18,936 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T12:30:18,938 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61319810, jitterRate=-0.08626362681388855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T12:30:18,939 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T12:30:18,941 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732192218895Writing region info on filesystem at 1732192218895Initializing all the Stores at 1732192218898 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218898Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218898Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192218898Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192218898Cleaning up temporary data from old regions at 1732192218932 (+34 ms)Running coprocessor post-open hooks at 1732192218939 (+7 ms)Region opened successfully at 1732192218941 (+2 ms) 2024-11-21T12:30:18,952 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732192218781 2024-11-21T12:30:18,970 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T12:30:18,970 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T12:30:18,973 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,977 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9bb5b7e9dd2c,41613,1732192216530, state=OPEN 2024-11-21T12:30:18,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:18,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:18,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:18,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:18,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,980 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:18,981 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:18,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T12:30:18,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,41613,1732192216530 in 371 msec 2024-11-21T12:30:18,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T12:30:18,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 864 msec 2024-11-21T12:30:19,000 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T12:30:19,000 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T12:30:19,026 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:19,028 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1] 2024-11-21T12:30:19,064 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T12:30:19,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53637, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T12:30:19,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2610 sec 2024-11-21T12:30:19,096 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732192219096, completionTime=-1 2024-11-21T12:30:19,099 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-21T12:30:19,099 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T12:30:19,172 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-21T12:30:19,172 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732192279172 2024-11-21T12:30:19,172 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732192339172 2024-11-21T12:30:19,172 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 72 msec 2024-11-21T12:30:19,174 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T12:30:19,181 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,182 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,182 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,184 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9bb5b7e9dd2c:32899, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,184 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,185 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,192 DEBUG [master/9bb5b7e9dd2c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T12:30:19,220 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.422sec 2024-11-21T12:30:19,222 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T12:30:19,224 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T12:30:19,225 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T12:30:19,226 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T12:30:19,226 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T12:30:19,227 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T12:30:19,227 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T12:30:19,232 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T12:30:19,233 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T12:30:19,234 INFO [master/9bb5b7e9dd2c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9bb5b7e9dd2c,32899,1732192215752-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:19,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287698dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T12:30:19,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9bb5b7e9dd2c,32899,-1 for getting cluster id 2024-11-21T12:30:19,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T12:30:19,265 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0ec7614b-bbbd-45b9-88cc-598d55c1d096' 2024-11-21T12:30:19,268 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T12:30:19,268 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0ec7614b-bbbd-45b9-88cc-598d55c1d096" 2024-11-21T12:30:19,269 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30900b79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T12:30:19,269 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9bb5b7e9dd2c,32899,-1] 2024-11-21T12:30:19,272 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T12:30:19,274 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:19,276 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T12:30:19,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd88229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T12:30:19,279 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:19,287 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1] 2024-11-21T12:30:19,288 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T12:30:19,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T12:30:19,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:19,315 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41319/hbase 2024-11-21T12:30:19,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=367, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=7476 2024-11-21T12:30:19,357 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:19,361 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:19,362 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:19,368 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-83974792, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-83974792, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:19,386 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-83974792/hregion-83974792.1732192219370, exclude list is [], retry=0 2024-11-21T12:30:19,392 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:19,392 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:19,393 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:19,431 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-83974792/hregion-83974792.1732192219370 2024-11-21T12:30:19,432 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:19,432 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 82e06bd2c125696e14ce7b9e51405dd2, NAME => 'testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741838_1014 (size=64) 2024-11-21T12:30:19,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741838_1014 (size=64) 2024-11-21T12:30:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741838_1014 (size=64) 2024-11-21T12:30:19,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:19,464 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,467 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82e06bd2c125696e14ce7b9e51405dd2 columnFamilyName a 2024-11-21T12:30:19,467 DEBUG [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:19,468 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] regionserver.HStore(327): Store=82e06bd2c125696e14ce7b9e51405dd2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:19,468 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,470 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,471 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,475 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,479 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:19,481 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82e06bd2c125696e14ce7b9e51405dd2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61470679, jitterRate=-0.08401550352573395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:19,482 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82e06bd2c125696e14ce7b9e51405dd2: Writing region info on filesystem at 1732192219461Initializing all the Stores at 1732192219463 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192219463Cleaning up temporary data from old regions at 1732192219472 (+9 ms)Region opened successfully at 1732192219482 (+10 ms) 2024-11-21T12:30:19,483 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 82e06bd2c125696e14ce7b9e51405dd2, disabling compactions & flushes 2024-11-21T12:30:19,483 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. 2024-11-21T12:30:19,483 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. 2024-11-21T12:30:19,483 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. after waiting 0 ms 2024-11-21T12:30:19,483 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. 2024-11-21T12:30:19,485 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. 2024-11-21T12:30:19,486 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 82e06bd2c125696e14ce7b9e51405dd2: Waiting for close lock at 1732192219483Disabling compacts and flushes for region at 1732192219483Disabling writes for close at 1732192219483Writing region close event to WAL at 1732192219485 (+2 ms)Closed at 1732192219485 2024-11-21T12:30:19,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741837_1013 (size=175) 2024-11-21T12:30:19,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741837_1013 (size=175) 2024-11-21T12:30:19,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741837_1013 (size=175) 2024-11-21T12:30:19,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:19,503 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-83974792:(num 1732192219370) 2024-11-21T12:30:19,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-21T12:30:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741839_1015 (size=469) 2024-11-21T12:30:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741839_1015 (size=469) 2024-11-21T12:30:19,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741839_1015 (size=469) 2024-11-21T12:30:19,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-21T12:30:19,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741840_1016 (size=322) 2024-11-21T12:30:19,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741840_1016 (size=322) 2024-11-21T12:30:19,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741840_1016 (size=322) 2024-11-21T12:30:19,634 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1, size=469 (469bytes) 2024-11-21T12:30:19,635 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-21T12:30:19,636 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-21T12:30:19,636 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1 2024-11-21T12:30:19,642 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1 after 4ms 2024-11-21T12:30:19,680 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:19,681 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1 took 48ms 2024-11-21T12:30:19,691 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1 so closing down 2024-11-21T12:30:19,692 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:19,694 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-21T12:30:19,730 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-21T12:30:19,732 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741841_1017 (size=469) 2024-11-21T12:30:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741841_1017 (size=469) 2024-11-21T12:30:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741841_1017 (size=469) 2024-11-21T12:30:19,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:19,750 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002 2024-11-21T12:30:19,754 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-21T12:30:19,754 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1, journal: Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1, size=469 (469bytes) at 1732192219634Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1 so closing down at 1732192219692 (+58 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp at 1732192219730 (+38 ms)3 split writer threads finished at 1732192219732 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732192219747 (+15 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002 at 1732192219750 (+3 ms)Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732192219754 (+4 ms) 2024-11-21T12:30:19,770 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2, size=322 (322bytes) 2024-11-21T12:30:19,771 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2 2024-11-21T12:30:19,772 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2 after 0ms 2024-11-21T12:30:19,803 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:19,804 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2 took 34ms 2024-11-21T12:30:19,807 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2 so closing down 2024-11-21T12:30:19,807 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:19,810 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-21T12:30:19,840 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-21T12:30:19,841 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741842_1018 (size=322) 2024-11-21T12:30:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741842_1018 (size=322) 2024-11-21T12:30:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741842_1018 (size=322) 2024-11-21T12:30:19,853 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:19,889 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:19,892 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-21T12:30:19,894 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-21T12:30:19,894 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2, journal: Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2, size=322 (322bytes) at 1732192219771Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2 so closing down at 1732192219807 (+36 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002-wal-2.temp at 1732192219840 (+33 ms)3 split writer threads finished at 1732192219841 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732192219853 (+12 ms)Processed 1 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732192219894 (+41 ms) 2024-11-21T12:30:19,894 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:19,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:19,912 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal.1732192219898, exclude list is [], retry=0 2024-11-21T12:30:19,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:19,918 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:19,918 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:19,949 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal.1732192219898 2024-11-21T12:30:19,950 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:19,950 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 82e06bd2c125696e14ce7b9e51405dd2, NAME => 'testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:19,950 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:19,950 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,950 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,953 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,955 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82e06bd2c125696e14ce7b9e51405dd2 columnFamilyName a 2024-11-21T12:30:19,955 DEBUG [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:19,956 INFO [StoreOpener-82e06bd2c125696e14ce7b9e51405dd2-1 {}] regionserver.HStore(327): Store=82e06bd2c125696e14ce7b9e51405dd2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:19,956 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,958 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:19,962 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002 2024-11-21T12:30:19,995 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:20,002 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002 2024-11-21T12:30:20,005 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 82e06bd2c125696e14ce7b9e51405dd2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-21T12:30:20,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/.tmp/a/7bc486ee5ef04fbcb2cc68a361e8494c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732192219503/Put/seqid=0 2024-11-21T12:30:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741844_1020 (size=5170) 2024-11-21T12:30:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741844_1020 (size=5170) 2024-11-21T12:30:20,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741844_1020 (size=5170) 2024-11-21T12:30:20,082 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/.tmp/a/7bc486ee5ef04fbcb2cc68a361e8494c 2024-11-21T12:30:20,146 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/.tmp/a/7bc486ee5ef04fbcb2cc68a361e8494c as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/a/7bc486ee5ef04fbcb2cc68a361e8494c 2024-11-21T12:30:20,161 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/a/7bc486ee5ef04fbcb2cc68a361e8494c, entries=2, sequenceid=2, filesize=5.0 K 2024-11-21T12:30:20,169 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 82e06bd2c125696e14ce7b9e51405dd2 in 160ms, sequenceid=2, compaction requested=false; wal=null 2024-11-21T12:30:20,172 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/0000000000000000002 2024-11-21T12:30:20,173 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:20,173 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:20,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 82e06bd2c125696e14ce7b9e51405dd2 2024-11-21T12:30:20,183 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/82e06bd2c125696e14ce7b9e51405dd2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-21T12:30:20,184 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 82e06bd2c125696e14ce7b9e51405dd2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70037148, jitterRate=0.04363483190536499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:20,186 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 82e06bd2c125696e14ce7b9e51405dd2: Writing region info on filesystem at 1732192219950Initializing all the Stores at 1732192219952 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192219952Obtaining lock to block concurrent updates at 1732192220005 (+53 ms)Preparing flush snapshotting stores in 82e06bd2c125696e14ce7b9e51405dd2 at 1732192220005Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732192220009 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732192219358.82e06bd2c125696e14ce7b9e51405dd2. at 1732192220009Flushing 82e06bd2c125696e14ce7b9e51405dd2/a: creating writer at 1732192220011 (+2 ms)Flushing 82e06bd2c125696e14ce7b9e51405dd2/a: appending metadata at 1732192220054 (+43 ms)Flushing 82e06bd2c125696e14ce7b9e51405dd2/a: closing flushed file at 1732192220057 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc93182: reopening flushed file at 1732192220142 (+85 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 82e06bd2c125696e14ce7b9e51405dd2 in 160ms, sequenceid=2, compaction requested=false; wal=null at 1732192220169 (+27 ms)Cleaning up temporary data from old regions at 1732192220173 (+4 ms)Region opened successfully at 1732192220185 (+12 ms) 2024-11-21T12:30:20,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 367) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40104 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:36864 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:58892 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59002 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:37032 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41319/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40034 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 204), ProcessCount=11 (was 11), AvailableMemoryMB=7450 (was 7476) 2024-11-21T12:30:20,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=7449 2024-11-21T12:30:20,250 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:20,253 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:20,254 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:20,258 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73403579, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-73403579, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:20,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73403579/hregion-73403579.1732192220259, exclude list is [], retry=0 2024-11-21T12:30:20,278 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:20,278 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:20,279 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:20,310 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73403579/hregion-73403579.1732192220259 2024-11-21T12:30:20,311 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:20,311 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e66475aa6d83e8d36f8256ef2e4a2ec2, NAME => 'testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741846_1022 (size=64) 2024-11-21T12:30:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741846_1022 (size=64) 2024-11-21T12:30:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741846_1022 (size=64) 2024-11-21T12:30:20,333 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:20,338 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,341 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e66475aa6d83e8d36f8256ef2e4a2ec2 columnFamilyName a 2024-11-21T12:30:20,341 DEBUG [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:20,342 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] regionserver.HStore(327): Store=e66475aa6d83e8d36f8256ef2e4a2ec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:20,343 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,344 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,345 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,345 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,345 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,348 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:20,353 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:20,354 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e66475aa6d83e8d36f8256ef2e4a2ec2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64774206, jitterRate=-0.03478911519050598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:20,354 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e66475aa6d83e8d36f8256ef2e4a2ec2: Writing region info on filesystem at 1732192220333Initializing all the Stores at 1732192220337 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192220337Cleaning up temporary data from old regions at 1732192220345 (+8 ms)Region opened successfully at 1732192220354 (+9 ms) 2024-11-21T12:30:20,354 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e66475aa6d83e8d36f8256ef2e4a2ec2, disabling compactions & flushes 2024-11-21T12:30:20,354 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. 2024-11-21T12:30:20,354 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. 2024-11-21T12:30:20,354 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. after waiting 0 ms 2024-11-21T12:30:20,354 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. 2024-11-21T12:30:20,355 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. 2024-11-21T12:30:20,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e66475aa6d83e8d36f8256ef2e4a2ec2: Waiting for close lock at 1732192220354Disabling compacts and flushes for region at 1732192220354Disabling writes for close at 1732192220354Writing region close event to WAL at 1732192220355 (+1 ms)Closed at 1732192220355 2024-11-21T12:30:20,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741845_1021 (size=175) 2024-11-21T12:30:20,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741845_1021 (size=175) 2024-11-21T12:30:20,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741845_1021 (size=175) 2024-11-21T12:30:20,365 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:20,365 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73403579:(num 1732192220259) 2024-11-21T12:30:20,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-21T12:30:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741847_1023 (size=469) 2024-11-21T12:30:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741847_1023 (size=469) 2024-11-21T12:30:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741847_1023 (size=469) 2024-11-21T12:30:20,821 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-21T12:30:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741848_1024 (size=322) 2024-11-21T12:30:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741848_1024 (size=322) 2024-11-21T12:30:20,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741848_1024 (size=322) 2024-11-21T12:30:20,881 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2, size=322 (322bytes) 2024-11-21T12:30:20,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2 2024-11-21T12:30:20,883 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2 after 1ms 2024-11-21T12:30:20,914 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:20,914 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2 took 33ms 2024-11-21T12:30:20,917 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2 so closing down 2024-11-21T12:30:20,917 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:20,920 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-21T12:30:20,952 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-21T12:30:20,952 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:20,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741849_1025 (size=322) 2024-11-21T12:30:20,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741849_1025 (size=322) 2024-11-21T12:30:20,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741849_1025 (size=322) 2024-11-21T12:30:20,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:20,965 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 2024-11-21T12:30:20,965 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-21T12:30:20,966 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2, journal: Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2, size=322 (322bytes) at 1732192220881Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2 so closing down at 1732192220917 (+36 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp at 1732192220952 (+35 ms)3 split writer threads finished at 1732192220952Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732192220963 (+11 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 at 1732192220965 (+2 ms)Processed 1 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732192220965 2024-11-21T12:30:20,983 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1, size=469 (469bytes) 2024-11-21T12:30:20,983 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1 2024-11-21T12:30:20,984 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1 after 1ms 2024-11-21T12:30:21,018 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:21,018 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1 took 36ms 2024-11-21T12:30:21,021 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1 so closing down 2024-11-21T12:30:21,021 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:21,023 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-21T12:30:21,054 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-21T12:30:21,054 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:21,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741850_1026 (size=469) 2024-11-21T12:30:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741850_1026 (size=469) 2024-11-21T12:30:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741850_1026 (size=469) 2024-11-21T12:30:21,065 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:21,097 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:21,100 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002, length=322 2024-11-21T12:30:21,103 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 2024-11-21T12:30:21,103 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-21T12:30:21,103 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1, journal: Splitting hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1, size=469 (469bytes) at 1732192220983Finishing writing output for hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1 so closing down at 1732192221021 (+38 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp at 1732192221054 (+33 ms)3 split writer threads finished at 1732192221054Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732192221065 (+11 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 at 1732192221103 (+38 ms)Processed 2 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732192221103 2024-11-21T12:30:21,104 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:21,106 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:21,126 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal.1732192221107, exclude list is [], retry=0 2024-11-21T12:30:21,130 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:21,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:21,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:21,162 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal.1732192221107 2024-11-21T12:30:21,163 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:21,163 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e66475aa6d83e8d36f8256ef2e4a2ec2, NAME => 'testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:21,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:21,164 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,164 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,166 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,168 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e66475aa6d83e8d36f8256ef2e4a2ec2 columnFamilyName a 2024-11-21T12:30:21,168 DEBUG [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:21,169 INFO [StoreOpener-e66475aa6d83e8d36f8256ef2e4a2ec2-1 {}] regionserver.HStore(327): Store=e66475aa6d83e8d36f8256ef2e4a2ec2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:21,170 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,171 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,174 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 2024-11-21T12:30:21,206 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:21,208 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 2024-11-21T12:30:21,208 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e66475aa6d83e8d36f8256ef2e4a2ec2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-21T12:30:21,227 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/.tmp/a/c2e234fa28d24720932f0db135f23cbe is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732192220365/Put/seqid=0 2024-11-21T12:30:21,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741852_1028 (size=5170) 2024-11-21T12:30:21,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741852_1028 (size=5170) 2024-11-21T12:30:21,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741852_1028 (size=5170) 2024-11-21T12:30:21,238 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/.tmp/a/c2e234fa28d24720932f0db135f23cbe 2024-11-21T12:30:21,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/.tmp/a/c2e234fa28d24720932f0db135f23cbe as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/a/c2e234fa28d24720932f0db135f23cbe 2024-11-21T12:30:21,262 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/a/c2e234fa28d24720932f0db135f23cbe, entries=2, sequenceid=2, filesize=5.0 K 2024-11-21T12:30:21,263 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e66475aa6d83e8d36f8256ef2e4a2ec2 in 55ms, sequenceid=2, compaction requested=false; wal=null 2024-11-21T12:30:21,264 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/0000000000000000002 2024-11-21T12:30:21,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,268 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e66475aa6d83e8d36f8256ef2e4a2ec2 2024-11-21T12:30:21,271 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/e66475aa6d83e8d36f8256ef2e4a2ec2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-21T12:30:21,272 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e66475aa6d83e8d36f8256ef2e4a2ec2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59092920, jitterRate=-0.11944687366485596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:21,272 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e66475aa6d83e8d36f8256ef2e4a2ec2: Writing region info on filesystem at 1732192221164Initializing all the Stores at 1732192221166 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192221166Obtaining lock to block concurrent updates at 1732192221208 (+42 ms)Preparing flush snapshotting stores in e66475aa6d83e8d36f8256ef2e4a2ec2 at 1732192221208Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732192221209 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732192220251.e66475aa6d83e8d36f8256ef2e4a2ec2. at 1732192221209Flushing e66475aa6d83e8d36f8256ef2e4a2ec2/a: creating writer at 1732192221209Flushing e66475aa6d83e8d36f8256ef2e4a2ec2/a: appending metadata at 1732192221226 (+17 ms)Flushing e66475aa6d83e8d36f8256ef2e4a2ec2/a: closing flushed file at 1732192221226Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@209f9663: reopening flushed file at 1732192221248 (+22 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e66475aa6d83e8d36f8256ef2e4a2ec2 in 55ms, sequenceid=2, compaction requested=false; wal=null at 1732192221263 (+15 ms)Cleaning up temporary data from old regions at 1732192221265 (+2 ms)Region opened successfully at 1732192221272 (+7 ms) 2024-11-21T12:30:21,293 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:36864 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:37112 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40034 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:58892 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40156 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59076 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 204), ProcessCount=11 (was 11), AvailableMemoryMB=7432 (was 7449) 2024-11-21T12:30:21,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=7431 2024-11-21T12:30:21,323 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:21,325 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:21,326 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:21,330 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73739849, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-73739849, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:21,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73739849/hregion-73739849.1732192221331, exclude list is [], retry=0 2024-11-21T12:30:21,350 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:21,350 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:21,351 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:21,382 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73739849/hregion-73739849.1732192221331 2024-11-21T12:30:21,382 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:21,382 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1c483c0cdcf62b00b567dd91b5119147, NAME => 'testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:21,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741854_1030 (size=64) 2024-11-21T12:30:21,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741854_1030 (size=64) 2024-11-21T12:30:21,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741854_1030 (size=64) 2024-11-21T12:30:21,396 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:21,398 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,400 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName a 2024-11-21T12:30:21,400 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:21,401 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:21,401 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,403 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName b 2024-11-21T12:30:21,403 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:21,403 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:21,403 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,405 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName c 2024-11-21T12:30:21,405 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:21,406 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:21,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,407 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,410 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,410 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:21,412 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:21,415 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:21,416 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c483c0cdcf62b00b567dd91b5119147; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58922411, jitterRate=-0.12198765575885773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:21,417 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c483c0cdcf62b00b567dd91b5119147: Writing region info on filesystem at 1732192221396Initializing all the Stores at 1732192221397 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192221398 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192221398Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192221398Cleaning up temporary data from old regions at 1732192221410 (+12 ms)Region opened successfully at 1732192221416 (+6 ms) 2024-11-21T12:30:21,417 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c483c0cdcf62b00b567dd91b5119147, disabling compactions & flushes 2024-11-21T12:30:21,417 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:21,417 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:21,417 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. after waiting 0 ms 2024-11-21T12:30:21,417 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:21,418 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:21,418 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c483c0cdcf62b00b567dd91b5119147: Waiting for close lock at 1732192221417Disabling compacts and flushes for region at 1732192221417Disabling writes for close at 1732192221417Writing region close event to WAL at 1732192221418 (+1 ms)Closed at 1732192221418 2024-11-21T12:30:21,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741853_1029 (size=175) 2024-11-21T12:30:21,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741853_1029 (size=175) 2024-11-21T12:30:21,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741853_1029 (size=175) 2024-11-21T12:30:21,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:21,425 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73739849:(num 1732192221331) 2024-11-21T12:30:21,425 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:21,428 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:21,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, exclude list is [], retry=0 2024-11-21T12:30:21,447 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:21,447 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:21,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:21,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 2024-11-21T12:30:21,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:21,685 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, size=0 (0bytes) 2024-11-21T12:30:21,685 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 might be still open, length is 0 2024-11-21T12:30:21,685 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 2024-11-21T12:30:21,687 WARN [IPC Server handler 4 on default port 41319 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-21T12:30:21,687 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 after 2ms 2024-11-21T12:30:23,529 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:37154 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34335:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37154 dst: /127.0.0.1:34335 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34335 remote=/127.0.0.1:37154]. Total timeout mills is 60000, 58120 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:23,530 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59134 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59134 dst: /127.0.0.1:39871 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:23,530 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40194 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37507:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40194 dst: /127.0.0.1:37507 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741855_1032 (size=497775) 2024-11-21T12:30:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741855_1032 (size=497775) 2024-11-21T12:30:23,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741855_1032 (size=497775) 2024-11-21T12:30:24,401 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T12:30:24,450 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T12:30:25,689 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 after 4004ms 2024-11-21T12:30:25,731 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:25,733 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 took 4048ms 2024-11-21T12:30:25,739 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732192221429.temp 2024-11-21T12:30:25,779 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp 2024-11-21T12:30:25,827 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429; continuing. 2024-11-21T12:30:25,828 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 so closing down 2024-11-21T12:30:25,828 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:25,828 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:25,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741856_1033 (size=497783) 2024-11-21T12:30:25,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741856_1033 (size=497783) 2024-11-21T12:30:25,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741856_1033 (size=497783) 2024-11-21T12:30:25,837 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp (wrote 3002 edits, skipped 0 edits in 39 ms) 2024-11-21T12:30:25,840 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 2024-11-21T12:30:25,840 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, size=0, length=0, corrupted=false, cancelled=false 2024-11-21T12:30:25,840 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, journal: Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, size=0 (0bytes) at 1732192221685Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp at 1732192225779 (+4094 ms)Split 1024 edits, skipped 0 edits. at 1732192225780 (+1 ms)Split 2048 edits, skipped 0 edits. at 1732192225807 (+27 ms)Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 so closing down at 1732192225828 (+21 ms)3 split writer threads finished at 1732192225828Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp (wrote 3002 edits, skipped 0 edits in 39 ms) at 1732192225837 (+9 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000000001-wal.1732192221429.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 at 1732192225840 (+3 ms)Processed 3002 edits across 1 Regions in 107 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429, size=0, length=0, corrupted=false, cancelled=false at 1732192225840 2024-11-21T12:30:25,843 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192221429 2024-11-21T12:30:25,844 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 2024-11-21T12:30:25,844 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:25,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:25,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192225847, exclude list is [], retry=0 2024-11-21T12:30:25,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:25,867 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:25,867 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:25,901 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192225847 2024-11-21T12:30:25,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:25,902 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:25,904 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,906 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName a 2024-11-21T12:30:25,906 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:25,906 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:25,907 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,908 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName b 2024-11-21T12:30:25,908 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:25,908 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:25,909 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,910 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c483c0cdcf62b00b567dd91b5119147 columnFamilyName c 2024-11-21T12:30:25,910 DEBUG [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:25,910 INFO [StoreOpener-1c483c0cdcf62b00b567dd91b5119147-1 {}] regionserver.HStore(327): Store=1c483c0cdcf62b00b567dd91b5119147/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:25,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:25,914 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 2024-11-21T12:30:25,946 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:25,978 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-21T12:30:26,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T12:30:26,208 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:26,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T12:30:26,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T12:30:26,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T12:30:26,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:26,211 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-21T12:30:26,211 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:26,400 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c483c0cdcf62b00b567dd91b5119147 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-21T12:30:26,444 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/a3e5e737371a470b9bceb80a357fb673 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732192221484/Put/seqid=0 2024-11-21T12:30:26,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741858_1035 (size=50463) 2024-11-21T12:30:26,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741858_1035 (size=50463) 2024-11-21T12:30:26,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741858_1035 (size=50463) 2024-11-21T12:30:26,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/a3e5e737371a470b9bceb80a357fb673 2024-11-21T12:30:26,476 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/a3e5e737371a470b9bceb80a357fb673 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/a/a3e5e737371a470b9bceb80a357fb673 2024-11-21T12:30:26,485 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/a/a3e5e737371a470b9bceb80a357fb673, entries=754, sequenceid=754, filesize=49.3 K 2024-11-21T12:30:26,486 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1c483c0cdcf62b00b567dd91b5119147 in 85ms, sequenceid=754, compaction requested=false; wal=null 2024-11-21T12:30:26,510 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-21T12:30:26,511 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c483c0cdcf62b00b567dd91b5119147 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-21T12:30:26,523 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/b1978cb7438b4b3184a08d507843a103 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732192221517/Put/seqid=0 2024-11-21T12:30:26,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741859_1036 (size=20072) 2024-11-21T12:30:26,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741859_1036 (size=20072) 2024-11-21T12:30:26,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741859_1036 (size=20072) 2024-11-21T12:30:26,536 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/b1978cb7438b4b3184a08d507843a103 2024-11-21T12:30:26,573 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/87c0152360354c1a98afd64b05ff2d20 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732192221554/Put/seqid=0 2024-11-21T12:30:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741860_1037 (size=35835) 2024-11-21T12:30:26,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741860_1037 (size=35835) 2024-11-21T12:30:26,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741860_1037 (size=35835) 2024-11-21T12:30:26,583 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/87c0152360354c1a98afd64b05ff2d20 2024-11-21T12:30:26,591 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/a/b1978cb7438b4b3184a08d507843a103 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/a/b1978cb7438b4b3184a08d507843a103 2024-11-21T12:30:26,599 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/a/b1978cb7438b4b3184a08d507843a103, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-21T12:30:26,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/87c0152360354c1a98afd64b05ff2d20 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/b/87c0152360354c1a98afd64b05ff2d20 2024-11-21T12:30:26,609 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/b/87c0152360354c1a98afd64b05ff2d20, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-21T12:30:26,609 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1c483c0cdcf62b00b567dd91b5119147 in 99ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-21T12:30:26,624 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-21T12:30:26,624 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c483c0cdcf62b00b567dd91b5119147 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-21T12:30:26,634 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/b3ec3dca4a054c6293ae0ca3155f67c6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732192221569/Put/seqid=0 2024-11-21T12:30:26,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741861_1038 (size=35082) 2024-11-21T12:30:26,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741861_1038 (size=35082) 2024-11-21T12:30:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741861_1038 (size=35082) 2024-11-21T12:30:26,647 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/b3ec3dca4a054c6293ae0ca3155f67c6 2024-11-21T12:30:26,676 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/0b80817705054f3aabcce6228adf33e3 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732192221601/Put/seqid=0 2024-11-21T12:30:26,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741862_1039 (size=20825) 2024-11-21T12:30:26,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741862_1039 (size=20825) 2024-11-21T12:30:26,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741862_1039 (size=20825) 2024-11-21T12:30:26,689 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/0b80817705054f3aabcce6228adf33e3 2024-11-21T12:30:26,698 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/b/b3ec3dca4a054c6293ae0ca3155f67c6 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/b/b3ec3dca4a054c6293ae0ca3155f67c6 2024-11-21T12:30:26,705 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/b/b3ec3dca4a054c6293ae0ca3155f67c6, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-21T12:30:26,706 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/0b80817705054f3aabcce6228adf33e3 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/c/0b80817705054f3aabcce6228adf33e3 2024-11-21T12:30:26,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/c/0b80817705054f3aabcce6228adf33e3, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-21T12:30:26,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1c483c0cdcf62b00b567dd91b5119147 in 89ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-21T12:30:26,724 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732192221646/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:26,727 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 2024-11-21T12:30:26,728 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-21T12:30:26,728 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c483c0cdcf62b00b567dd91b5119147 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-21T12:30:26,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/41dcd40ad816436c8650e34e5a9d8fd5 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732192221611/Put/seqid=0 2024-11-21T12:30:26,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741863_1040 (size=50301) 2024-11-21T12:30:26,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741863_1040 (size=50301) 2024-11-21T12:30:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741863_1040 (size=50301) 2024-11-21T12:30:26,751 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/41dcd40ad816436c8650e34e5a9d8fd5 2024-11-21T12:30:26,759 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41dcd40ad816436c8650e34e5a9d8fd5 2024-11-21T12:30:26,760 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/.tmp/c/41dcd40ad816436c8650e34e5a9d8fd5 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/c/41dcd40ad816436c8650e34e5a9d8fd5 2024-11-21T12:30:26,767 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41dcd40ad816436c8650e34e5a9d8fd5 2024-11-21T12:30:26,767 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/c/41dcd40ad816436c8650e34e5a9d8fd5, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-21T12:30:26,768 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1c483c0cdcf62b00b567dd91b5119147 in 40ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-21T12:30:26,769 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/0000000000000003002 2024-11-21T12:30:26,770 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:26,770 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:26,771 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T12:30:26,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c483c0cdcf62b00b567dd91b5119147 2024-11-21T12:30:26,775 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenIntoWAL/1c483c0cdcf62b00b567dd91b5119147/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-21T12:30:26,777 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c483c0cdcf62b00b567dd91b5119147; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60088878, jitterRate=-0.10460594296455383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T12:30:26,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c483c0cdcf62b00b567dd91b5119147: Writing region info on filesystem at 1732192225902Initializing all the Stores at 1732192225904 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192225904Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192225904Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192225904Cleaning up temporary data from old regions at 1732192226770 (+866 ms)Region opened successfully at 1732192226777 (+7 ms) 2024-11-21T12:30:26,830 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c483c0cdcf62b00b567dd91b5119147, disabling compactions & flushes 2024-11-21T12:30:26,830 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:26,830 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:26,830 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. after waiting 0 ms 2024-11-21T12:30:26,830 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:26,832 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732192221323.1c483c0cdcf62b00b567dd91b5119147. 2024-11-21T12:30:26,832 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c483c0cdcf62b00b567dd91b5119147: Waiting for close lock at 1732192226829Disabling compacts and flushes for region at 1732192226829Disabling writes for close at 1732192226830 (+1 ms)Writing region close event to WAL at 1732192226832 (+2 ms)Closed at 1732192226832 2024-11-21T12:30:26,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741857_1034 (size=175) 2024-11-21T12:30:26,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741857_1034 (size=175) 2024-11-21T12:30:26,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741857_1034 (size=175) 2024-11-21T12:30:26,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:26,843 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732192225847) 2024-11-21T12:30:26,866 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 388) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:37441 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_538859568_22 at /127.0.0.1:37194 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37147 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_538859568_22 at /127.0.0.1:59164 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41319 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_538859568_22 at /127.0.0.1:40240 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37441 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41319 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:37147 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_538859568_22 at /127.0.0.1:40252 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 204), ProcessCount=11 (was 11), AvailableMemoryMB=7326 (was 7431) 2024-11-21T12:30:26,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=7324 2024-11-21T12:30:26,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:26,902 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:26,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:26,907 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67672330, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-67672330, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:26,922 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67672330/hregion-67672330.1732192226908, exclude list is [], retry=0 2024-11-21T12:30:26,926 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:26,926 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:26,926 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:26,959 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67672330/hregion-67672330.1732192226908 2024-11-21T12:30:26,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:26,960 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 94086d09c01d9ef38f96d7f24e40feb2, NAME => 'test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741865_1042 (size=43) 2024-11-21T12:30:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741865_1042 (size=43) 2024-11-21T12:30:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741865_1042 (size=43) 2024-11-21T12:30:26,975 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:26,977 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,979 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName a 2024-11-21T12:30:26,979 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:26,980 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:26,980 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,982 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName b 2024-11-21T12:30:26,982 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:26,983 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:26,983 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,985 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName c 2024-11-21T12:30:26,985 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:26,986 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:26,986 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,989 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,989 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,990 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:26,991 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:26,994 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:26,994 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94086d09c01d9ef38f96d7f24e40feb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67208958, jitterRate=0.0014915168285369873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94086d09c01d9ef38f96d7f24e40feb2: Writing region info on filesystem at 1732192226975Initializing all the Stores at 1732192226977 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192226977Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192226977Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192226977Cleaning up temporary data from old regions at 1732192226989 (+12 ms)Region opened successfully at 1732192226996 (+7 ms) 2024-11-21T12:30:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94086d09c01d9ef38f96d7f24e40feb2, disabling compactions & flushes 2024-11-21T12:30:26,996 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. after waiting 0 ms 2024-11-21T12:30:26,996 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:26,997 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:26,997 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94086d09c01d9ef38f96d7f24e40feb2: Waiting for close lock at 1732192226996Disabling compacts and flushes for region at 1732192226996Disabling writes for close at 1732192226996Writing region close event to WAL at 1732192226997 (+1 ms)Closed at 1732192226997 2024-11-21T12:30:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741864_1041 (size=175) 2024-11-21T12:30:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741864_1041 (size=175) 2024-11-21T12:30:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741864_1041 (size=175) 2024-11-21T12:30:27,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:27,003 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67672330:(num 1732192226908) 2024-11-21T12:30:27,003 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:27,006 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:27,024 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, exclude list is [], retry=0 2024-11-21T12:30:27,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:27,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:27,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:27,060 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 2024-11-21T12:30:27,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:27,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741866_1043 (size=371388) 2024-11-21T12:30:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741866_1043 (size=371388) 2024-11-21T12:30:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741866_1043 (size=371388) 2024-11-21T12:30:27,226 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, size=362.7 K (371388bytes) 2024-11-21T12:30:27,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 2024-11-21T12:30:27,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 after 0ms 2024-11-21T12:30:27,260 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:27,261 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 took 35ms 2024-11-21T12:30:27,267 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732192227006.temp 2024-11-21T12:30:27,304 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp 2024-11-21T12:30:27,309 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 so closing down 2024-11-21T12:30:27,309 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:27,323 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741867_1044 (size=371388) 2024-11-21T12:30:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741867_1044 (size=371388) 2024-11-21T12:30:27,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741867_1044 (size=371388) 2024-11-21T12:30:27,328 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-21T12:30:27,330 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp to hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 2024-11-21T12:30:27,330 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-21T12:30:27,330 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, journal: Splitting hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, size=362.7 K (371388bytes) at 1732192227226Split 1024 edits, skipped 0 edits. at 1732192227278 (+52 ms)Split 2048 edits, skipped 0 edits. at 1732192227294 (+16 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp at 1732192227305 (+11 ms)Finishing writing output for hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 so closing down at 1732192227309 (+4 ms)3 split writer threads finished at 1732192227323 (+14 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1732192227328 (+5 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000000001-wal.1732192227006.temp to hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 at 1732192227330 (+2 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732192227330 2024-11-21T12:30:27,332 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227006 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192227006 2024-11-21T12:30:27,334 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 2024-11-21T12:30:27,334 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:27,336 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:27,358 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, exclude list is [], retry=0 2024-11-21T12:30:27,362 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:27,362 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:27,362 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:27,394 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 2024-11-21T12:30:27,394 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741868_1045 (size=371515) 2024-11-21T12:30:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741868_1045 (size=371515) 2024-11-21T12:30:27,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741868_1045 (size=371515) 2024-11-21T12:30:27,592 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, size=362.8 K (371515bytes) 2024-11-21T12:30:27,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 2024-11-21T12:30:27,593 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 after 1ms 2024-11-21T12:30:27,625 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:27,627 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 took 35ms 2024-11-21T12:30:27,632 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732192227337.temp 2024-11-21T12:30:27,689 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 so closing down 2024-11-21T12:30:27,689 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:27,705 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp 2024-11-21T12:30:27,729 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741869_1046 (size=371515) 2024-11-21T12:30:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741869_1046 (size=371515) 2024-11-21T12:30:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741869_1046 (size=371515) 2024-11-21T12:30:27,734 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp (wrote 3000 edits, skipped 0 edits in 23 ms) 2024-11-21T12:30:27,736 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp to hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 2024-11-21T12:30:27,736 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 109 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-21T12:30:27,737 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, journal: Splitting hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, size=362.8 K (371515bytes) at 1732192227592Split 1024 edits, skipped 0 edits. at 1732192227671 (+79 ms)Split 2048 edits, skipped 0 edits. at 1732192227681 (+10 ms)Finishing writing output for hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 so closing down at 1732192227689 (+8 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp at 1732192227705 (+16 ms)3 split writer threads finished at 1732192227729 (+24 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp (wrote 3000 edits, skipped 0 edits in 23 ms) at 1732192227734 (+5 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003001-wal.1732192227337.temp to hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 at 1732192227736 (+2 ms)Processed 3000 edits across 1 Regions in 109 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732192227736 2024-11-21T12:30:27,739 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227337 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192227337 2024-11-21T12:30:27,740 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 2024-11-21T12:30:27,740 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:27,742 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/test2727-manual,16010,1732192226898, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:27,764 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227743, exclude list is [], retry=0 2024-11-21T12:30:27,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:27,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:27,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:27,810 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227743 2024-11-21T12:30:27,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:27,810 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 94086d09c01d9ef38f96d7f24e40feb2, NAME => 'test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,811 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,813 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,814 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName a 2024-11-21T12:30:27,814 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:27,815 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:27,815 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,816 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName b 2024-11-21T12:30:27,816 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:27,817 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:27,817 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,818 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94086d09c01d9ef38f96d7f24e40feb2 columnFamilyName c 2024-11-21T12:30:27,818 DEBUG [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:27,818 INFO [StoreOpener-94086d09c01d9ef38f96d7f24e40feb2-1 {}] regionserver.HStore(327): Store=94086d09c01d9ef38f96d7f24e40feb2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:27,818 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,819 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,821 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:27,822 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 2024-11-21T12:30:27,852 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:27,875 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 2024-11-21T12:30:27,877 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 2024-11-21T12:30:27,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:27,930 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 2024-11-21T12:30:27,930 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 94086d09c01d9ef38f96d7f24e40feb2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-21T12:30:27,962 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/a/71366bd65d0943fcaf8d7a9b15f0b0b4 is 41, key is test2727/a:100/1732192227400/Put/seqid=0 2024-11-21T12:30:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741871_1048 (size=84227) 2024-11-21T12:30:27,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741871_1048 (size=84227) 2024-11-21T12:30:27,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741871_1048 (size=84227) 2024-11-21T12:30:27,970 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/a/71366bd65d0943fcaf8d7a9b15f0b0b4 2024-11-21T12:30:28,000 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/b/07aea089379d4e20908a43add7ed2376 is 41, key is test2727/b:100/1732192227470/Put/seqid=0 2024-11-21T12:30:28,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741872_1049 (size=84609) 2024-11-21T12:30:28,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741872_1049 (size=84609) 2024-11-21T12:30:28,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741872_1049 (size=84609) 2024-11-21T12:30:28,008 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/b/07aea089379d4e20908a43add7ed2376 2024-11-21T12:30:28,044 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/c/a8e09d1883c14444895333276fb5041f is 41, key is test2727/c:100/1732192227525/Put/seqid=0 2024-11-21T12:30:28,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741873_1050 (size=84609) 2024-11-21T12:30:28,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741873_1050 (size=84609) 2024-11-21T12:30:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741873_1050 (size=84609) 2024-11-21T12:30:28,051 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/c/a8e09d1883c14444895333276fb5041f 2024-11-21T12:30:28,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/a/71366bd65d0943fcaf8d7a9b15f0b0b4 as hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/a/71366bd65d0943fcaf8d7a9b15f0b0b4 2024-11-21T12:30:28,068 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/a/71366bd65d0943fcaf8d7a9b15f0b0b4, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-21T12:30:28,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/b/07aea089379d4e20908a43add7ed2376 as hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/b/07aea089379d4e20908a43add7ed2376 2024-11-21T12:30:28,077 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/b/07aea089379d4e20908a43add7ed2376, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-21T12:30:28,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/.tmp/c/a8e09d1883c14444895333276fb5041f as hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/c/a8e09d1883c14444895333276fb5041f 2024-11-21T12:30:28,085 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/c/a8e09d1883c14444895333276fb5041f, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-21T12:30:28,085 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 94086d09c01d9ef38f96d7f24e40feb2 in 155ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-21T12:30:28,086 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000003000 2024-11-21T12:30:28,087 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/0000000000000006000 2024-11-21T12:30:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:28,089 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:28,091 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94086d09c01d9ef38f96d7f24e40feb2 2024-11-21T12:30:28,094 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/test2727/94086d09c01d9ef38f96d7f24e40feb2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-21T12:30:28,095 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94086d09c01d9ef38f96d7f24e40feb2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65753042, jitterRate=-0.020203322172164917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:28,096 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94086d09c01d9ef38f96d7f24e40feb2: Writing region info on filesystem at 1732192227811Initializing all the Stores at 1732192227812 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192227812Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192227813 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192227813Obtaining lock to block concurrent updates at 1732192227931 (+118 ms)Preparing flush snapshotting stores in 94086d09c01d9ef38f96d7f24e40feb2 at 1732192227931Finished memstore snapshotting test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732192227931Flushing stores of test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. at 1732192227931Flushing 94086d09c01d9ef38f96d7f24e40feb2/a: creating writer at 1732192227931Flushing 94086d09c01d9ef38f96d7f24e40feb2/a: appending metadata at 1732192227961 (+30 ms)Flushing 94086d09c01d9ef38f96d7f24e40feb2/a: closing flushed file at 1732192227961Flushing 94086d09c01d9ef38f96d7f24e40feb2/b: creating writer at 1732192227977 (+16 ms)Flushing 94086d09c01d9ef38f96d7f24e40feb2/b: appending metadata at 1732192227998 (+21 ms)Flushing 94086d09c01d9ef38f96d7f24e40feb2/b: closing flushed file at 1732192227998Flushing 94086d09c01d9ef38f96d7f24e40feb2/c: creating writer at 1732192228014 (+16 ms)Flushing 94086d09c01d9ef38f96d7f24e40feb2/c: appending metadata at 1732192228043 (+29 ms)Flushing 94086d09c01d9ef38f96d7f24e40feb2/c: closing flushed file at 1732192228043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b93098: reopening flushed file at 1732192228059 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f42c743: reopening flushed file at 1732192228068 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d48108d: reopening flushed file at 1732192228077 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 94086d09c01d9ef38f96d7f24e40feb2 in 155ms, sequenceid=6000, compaction requested=false; wal=null at 1732192228085 (+8 ms)Cleaning up temporary data from old regions at 1732192228088 (+3 ms)Region opened successfully at 1732192228096 (+8 ms) 2024-11-21T12:30:28,098 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-21T12:30:28,098 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94086d09c01d9ef38f96d7f24e40feb2, disabling compactions & flushes 2024-11-21T12:30:28,098 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:28,098 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:28,098 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. after waiting 0 ms 2024-11-21T12:30:28,098 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:28,100 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732192226900.94086d09c01d9ef38f96d7f24e40feb2. 2024-11-21T12:30:28,100 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94086d09c01d9ef38f96d7f24e40feb2: Waiting for close lock at 1732192228098Disabling compacts and flushes for region at 1732192228098Disabling writes for close at 1732192228098Writing region close event to WAL at 1732192228100 (+2 ms)Closed at 1732192228100 2024-11-21T12:30:28,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741870_1047 (size=175) 2024-11-21T12:30:28,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741870_1047 (size=175) 2024-11-21T12:30:28,104 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732192226898/wal.1732192227743 not finished, retry = 0 2024-11-21T12:30:28,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741870_1047 (size=175) 2024-11-21T12:30:28,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:28,207 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732192227743) 2024-11-21T12:30:28,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=406 (was 404) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59536 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:42346 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40252 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=7100 (was 7324) 2024-11-21T12:30:28,233 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=7099 2024-11-21T12:30:28,249 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:28,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:28,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732192228256 2024-11-21T12:30:28,290 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 2024-11-21T12:30:28,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:28,294 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d727fc8d89cf2e83404153a0535c185, NAME => 'testSequentialEditLogSeqNum,,1732192228250.6d727fc8d89cf2e83404153a0535c185.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:28,294 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732192228250.6d727fc8d89cf2e83404153a0535c185.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:28,294 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,294 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,295 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185 doesn't exist for region: 6d727fc8d89cf2e83404153a0535c185 on table testSequentialEditLogSeqNum 2024-11-21T12:30:28,296 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 6d727fc8d89cf2e83404153a0535c185 on table testSequentialEditLogSeqNum 2024-11-21T12:30:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741875_1052 (size=62) 2024-11-21T12:30:28,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741875_1052 (size=62) 2024-11-21T12:30:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741875_1052 (size=62) 2024-11-21T12:30:28,308 INFO [StoreOpener-6d727fc8d89cf2e83404153a0535c185-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,309 INFO [StoreOpener-6d727fc8d89cf2e83404153a0535c185-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d727fc8d89cf2e83404153a0535c185 columnFamilyName a 2024-11-21T12:30:28,309 DEBUG [StoreOpener-6d727fc8d89cf2e83404153a0535c185-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,310 INFO [StoreOpener-6d727fc8d89cf2e83404153a0535c185-1 {}] regionserver.HStore(327): Store=6d727fc8d89cf2e83404153a0535c185/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,310 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,312 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,312 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,314 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6d727fc8d89cf2e83404153a0535c185 2024-11-21T12:30:28,317 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:28,317 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6d727fc8d89cf2e83404153a0535c185; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61578828, jitterRate=-0.08240395784378052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:28,318 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6d727fc8d89cf2e83404153a0535c185: Writing region info on filesystem at 1732192228294Initializing all the Stores at 1732192228307 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228307Cleaning up temporary data from old regions at 1732192228312 (+5 ms)Region opened successfully at 1732192228318 (+6 ms) 2024-11-21T12:30:28,330 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6d727fc8d89cf2e83404153a0535c185 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-21T12:30:28,353 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/.tmp/a/9b63629652064861bc0dd2bbe772d0f4 is 81, key is testSequentialEditLogSeqNum/a:x0/1732192228318/Put/seqid=0 2024-11-21T12:30:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741876_1053 (size=5833) 2024-11-21T12:30:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741876_1053 (size=5833) 2024-11-21T12:30:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741876_1053 (size=5833) 2024-11-21T12:30:28,362 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/.tmp/a/9b63629652064861bc0dd2bbe772d0f4 2024-11-21T12:30:28,368 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/.tmp/a/9b63629652064861bc0dd2bbe772d0f4 as hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/a/9b63629652064861bc0dd2bbe772d0f4 2024-11-21T12:30:28,374 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/a/9b63629652064861bc0dd2bbe772d0f4, entries=10, sequenceid=13, filesize=5.7 K 2024-11-21T12:30:28,376 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6d727fc8d89cf2e83404153a0535c185 in 47ms, sequenceid=13, compaction requested=false 2024-11-21T12:30:28,376 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6d727fc8d89cf2e83404153a0535c185: 2024-11-21T12:30:28,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-21T12:30:28,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-21T12:30:28,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-21T12:30:28,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-21T12:30:28,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-21T12:30:28,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741874_1051 (size=3256) 2024-11-21T12:30:28,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741874_1051 (size=3256) 2024-11-21T12:30:28,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741874_1051 (size=3256) 2024-11-21T12:30:28,402 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256, size=3.2 K (3256bytes) 2024-11-21T12:30:28,402 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 2024-11-21T12:30:28,403 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 after 1ms 2024-11-21T12:30:28,432 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:28,432 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 took 30ms 2024-11-21T12:30:28,436 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 so closing down 2024-11-21T12:30:28,436 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:28,437 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732192228256.temp 2024-11-21T12:30:28,464 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp 2024-11-21T12:30:28,465 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741877_1054 (size=2650) 2024-11-21T12:30:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741877_1054 (size=2650) 2024-11-21T12:30:28,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741877_1054 (size=2650) 2024-11-21T12:30:28,474 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:28,476 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp to hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000020 2024-11-21T12:30:28,476 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 44 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-21T12:30:28,477 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256, journal: Splitting hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256, size=3.2 K (3256bytes) at 1732192228402Finishing writing output for hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256 so closing down at 1732192228436 (+34 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp at 1732192228464 (+28 ms)3 split writer threads finished at 1732192228465 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732192228474 (+9 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000003-wal.1732192228256.temp to hdfs://localhost:41319/hbase/data/default/testSequentialEditLogSeqNum/6d727fc8d89cf2e83404153a0535c185/recovered.edits/0000000000000000020 at 1732192228476 (+2 ms)Processed 17 edits across 1 Regions in 44 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732192228248/wal.1732192228256, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732192228477 (+1 ms) 2024-11-21T12:30:28,495 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 406) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59536 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:42346 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=959 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=7096 (was 7099) 2024-11-21T12:30:28,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=959, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=7096 2024-11-21T12:30:28,532 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:28,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:28,561 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:28,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98466354, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-98466354, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:28,579 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98466354/hregion-98466354.1732192228565, exclude list is [], retry=0 2024-11-21T12:30:28,583 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:28,583 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:28,583 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:28,614 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98466354/hregion-98466354.1732192228565 2024-11-21T12:30:28,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:28,615 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7560e111d10d5bc0489ebb297aab54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:28,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741879_1056 (size=70) 2024-11-21T12:30:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741879_1056 (size=70) 2024-11-21T12:30:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741879_1056 (size=70) 2024-11-21T12:30:28,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:28,635 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,637 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName a 2024-11-21T12:30:28,637 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,638 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,638 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,640 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName b 2024-11-21T12:30:28,640 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,641 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,641 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,642 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName c 2024-11-21T12:30:28,643 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,643 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,643 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,644 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,646 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,646 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,647 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:28,648 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,651 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:28,652 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7560e111d10d5bc0489ebb297aab54bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58953043, jitterRate=-0.12153120338916779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:28,653 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7560e111d10d5bc0489ebb297aab54bb: Writing region info on filesystem at 1732192228634Initializing all the Stores at 1732192228635 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228635Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228635Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228635Cleaning up temporary data from old regions at 1732192228646 (+11 ms)Region opened successfully at 1732192228653 (+7 ms) 2024-11-21T12:30:28,653 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7560e111d10d5bc0489ebb297aab54bb, disabling compactions & flushes 2024-11-21T12:30:28,653 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:28,653 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:28,653 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. after waiting 0 ms 2024-11-21T12:30:28,653 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:28,654 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:28,654 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7560e111d10d5bc0489ebb297aab54bb: Waiting for close lock at 1732192228653Disabling compacts and flushes for region at 1732192228653Disabling writes for close at 1732192228653Writing region close event to WAL at 1732192228654 (+1 ms)Closed at 1732192228654 2024-11-21T12:30:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741878_1055 (size=175) 2024-11-21T12:30:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741878_1055 (size=175) 2024-11-21T12:30:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741878_1055 (size=175) 2024-11-21T12:30:28,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:28,664 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98466354:(num 1732192228565) 2024-11-21T12:30:28,664 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:28,666 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:28,684 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, exclude list is [], retry=0 2024-11-21T12:30:28,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:28,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:28,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:28,721 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 2024-11-21T12:30:28,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:28,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7560e111d10d5bc0489ebb297aab54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:28,722 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:28,722 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,722 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,724 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,725 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName a 2024-11-21T12:30:28,725 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,726 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,726 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,727 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName b 2024-11-21T12:30:28,727 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,727 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,727 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,728 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName c 2024-11-21T12:30:28,728 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:28,728 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:28,729 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,729 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,730 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,732 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,732 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,732 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:28,734 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:28,735 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7560e111d10d5bc0489ebb297aab54bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67707178, jitterRate=0.008915573358535767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:28,736 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7560e111d10d5bc0489ebb297aab54bb: Writing region info on filesystem at 1732192228722Initializing all the Stores at 1732192228723 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228723Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228724 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192228724Cleaning up temporary data from old regions at 1732192228732 (+8 ms)Region opened successfully at 1732192228736 (+4 ms) 2024-11-21T12:30:28,740 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732192228739/Put/seqid=0 2024-11-21T12:30:28,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741881_1058 (size=4826) 2024-11-21T12:30:28,749 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41319/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 7560e111d10d5bc0489ebb297aab54bb/a 2024-11-21T12:30:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741881_1058 (size=4826) 2024-11-21T12:30:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741881_1058 (size=4826) 2024-11-21T12:30:28,757 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-21T12:30:28,757 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-21T12:30:28,758 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7560e111d10d5bc0489ebb297aab54bb: 2024-11-21T12:30:28,759 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_ 2024-11-21T12:30:28,760 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41319/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7560e111d10d5bc0489ebb297aab54bb/a as hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_ - updating store file list. 2024-11-21T12:30:28,766 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:28,767 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_ into 7560e111d10d5bc0489ebb297aab54bb/a 2024-11-21T12:30:28,767 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41319/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7560e111d10d5bc0489ebb297aab54bb/a (new location: hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_) 2024-11-21T12:30:28,813 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, size=0 (0bytes) 2024-11-21T12:30:28,813 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 might be still open, length is 0 2024-11-21T12:30:28,813 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 2024-11-21T12:30:28,814 WARN [IPC Server handler 2 on default port 41319 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-21T12:30:28,814 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 after 1ms 2024-11-21T12:30:29,510 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59656 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59656 dst: /127.0.0.1:39871 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39871 remote=/127.0.0.1:59656]. Total timeout mills is 60000, 59263 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:29,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:50350 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37507:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50350 dst: /127.0.0.1:37507 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:29,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:42450 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34335:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42450 dst: /127.0.0.1:34335 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:29,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741880_1059 (size=606) 2024-11-21T12:30:29,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741880_1059 (size=606) 2024-11-21T12:30:32,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 after 4002ms 2024-11-21T12:30:32,847 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:32,847 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 took 4034ms 2024-11-21T12:30:32,850 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667; continuing. 2024-11-21T12:30:32,850 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 so closing down 2024-11-21T12:30:32,850 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:32,852 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732192228667.temp 2024-11-21T12:30:32,887 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp 2024-11-21T12:30:32,887 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741882_1060 (size=329) 2024-11-21T12:30:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741882_1060 (size=329) 2024-11-21T12:30:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741882_1060 (size=329) 2024-11-21T12:30:32,898 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:32,900 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp to hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 2024-11-21T12:30:32,901 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 54 ms; skipped=1; WAL=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, size=0, length=0, corrupted=false, cancelled=false 2024-11-21T12:30:32,901 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, journal: Splitting hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, size=0 (0bytes) at 1732192228813Finishing writing output for hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 so closing down at 1732192232850 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp at 1732192232887 (+37 ms)3 split writer threads finished at 1732192232887Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732192232899 (+12 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005-wal.1732192228667.temp to hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 at 1732192232901 (+2 ms)Processed 2 edits across 1 Regions in 54 ms; skipped=1; WAL=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667, size=0, length=0, corrupted=false, cancelled=false at 1732192232901 2024-11-21T12:30:32,903 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192228667 2024-11-21T12:30:32,905 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 2024-11-21T12:30:32,905 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:32,907 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:32,922 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192232907, exclude list is [], retry=0 2024-11-21T12:30:32,925 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:32,926 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:32,926 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:32,958 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192232907 2024-11-21T12:30:32,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:32,959 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7560e111d10d5bc0489ebb297aab54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:32,959 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:32,960 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,960 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,962 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,963 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName a 2024-11-21T12:30:32,963 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:32,971 DEBUG [StoreFileOpener-7560e111d10d5bc0489ebb297aab54bb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:32,971 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/ef84df0f999049d29ba3fa48b04feb9b_SeqId_3_ 2024-11-21T12:30:32,971 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:32,971 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,973 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName b 2024-11-21T12:30:32,973 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:32,973 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:32,973 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,974 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7560e111d10d5bc0489ebb297aab54bb columnFamilyName c 2024-11-21T12:30:32,974 DEBUG [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:32,975 INFO [StoreOpener-7560e111d10d5bc0489ebb297aab54bb-1 {}] regionserver.HStore(327): Store=7560e111d10d5bc0489ebb297aab54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:32,975 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,976 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:32,978 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 2024-11-21T12:30:33,009 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:33,011 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 2024-11-21T12:30:33,011 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7560e111d10d5bc0489ebb297aab54bb 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-21T12:30:33,029 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/.tmp/a/c1cf47acb4b54b19b838fee526dc2a71 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732192228772/Put/seqid=0 2024-11-21T12:30:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741884_1062 (size=5149) 2024-11-21T12:30:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741884_1062 (size=5149) 2024-11-21T12:30:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741884_1062 (size=5149) 2024-11-21T12:30:33,039 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/.tmp/a/c1cf47acb4b54b19b838fee526dc2a71 2024-11-21T12:30:33,046 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/.tmp/a/c1cf47acb4b54b19b838fee526dc2a71 as hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/c1cf47acb4b54b19b838fee526dc2a71 2024-11-21T12:30:33,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/a/c1cf47acb4b54b19b838fee526dc2a71, entries=1, sequenceid=5, filesize=5.0 K 2024-11-21T12:30:33,053 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7560e111d10d5bc0489ebb297aab54bb in 42ms, sequenceid=5, compaction requested=false; wal=null 2024-11-21T12:30:33,054 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/0000000000000000005 2024-11-21T12:30:33,055 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:33,055 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:33,056 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:33,058 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7560e111d10d5bc0489ebb297aab54bb 2024-11-21T12:30:33,060 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7560e111d10d5bc0489ebb297aab54bb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-21T12:30:33,061 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7560e111d10d5bc0489ebb297aab54bb; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59030391, jitterRate=-0.12037862837314606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:33,062 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7560e111d10d5bc0489ebb297aab54bb: Writing region info on filesystem at 1732192232960Initializing all the Stores at 1732192232961 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192232961Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192232961Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192232961Obtaining lock to block concurrent updates at 1732192233011 (+50 ms)Preparing flush snapshotting stores in 7560e111d10d5bc0489ebb297aab54bb at 1732192233011Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732192233011Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. at 1732192233011Flushing 7560e111d10d5bc0489ebb297aab54bb/a: creating writer at 1732192233011Flushing 7560e111d10d5bc0489ebb297aab54bb/a: appending metadata at 1732192233029 (+18 ms)Flushing 7560e111d10d5bc0489ebb297aab54bb/a: closing flushed file at 1732192233029Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fb562ca: reopening flushed file at 1732192233045 (+16 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7560e111d10d5bc0489ebb297aab54bb in 42ms, sequenceid=5, compaction requested=false; wal=null at 1732192233053 (+8 ms)Cleaning up temporary data from old regions at 1732192233055 (+2 ms)Region opened successfully at 1732192233062 (+7 ms) 2024-11-21T12:30:33,067 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7560e111d10d5bc0489ebb297aab54bb, disabling compactions & flushes 2024-11-21T12:30:33,067 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:33,067 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:33,067 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. after waiting 0 ms 2024-11-21T12:30:33,067 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:33,068 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732192228533.7560e111d10d5bc0489ebb297aab54bb. 2024-11-21T12:30:33,068 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7560e111d10d5bc0489ebb297aab54bb: Waiting for close lock at 1732192233067Disabling compacts and flushes for region at 1732192233067Disabling writes for close at 1732192233067Writing region close event to WAL at 1732192233068 (+1 ms)Closed at 1732192233068 2024-11-21T12:30:33,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741883_1061 (size=175) 2024-11-21T12:30:33,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741883_1061 (size=175) 2024-11-21T12:30:33,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741883_1061 (size=175) 2024-11-21T12:30:33,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:33,074 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732192232907) 2024-11-21T12:30:33,090 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=416 (was 409) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41319 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41319 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1147270389_22 at /127.0.0.1:42476 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1147270389_22 at /127.0.0.1:59676 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1147270389_22 at /127.0.0.1:50388 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 959) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=7097 (was 7096) - AvailableMemoryMB LEAK? - 2024-11-21T12:30:33,104 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=416, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=7097 2024-11-21T12:30:33,121 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:33,127 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T12:30:33,132 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:33,135 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a2c39bb 2024-11-21T12:30:33,136 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T12:30:33,139 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T12:30:33,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T12:30:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-21T12:30:33,153 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T12:30:33,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-21T12:30:33,156 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:33,158 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T12:30:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T12:30:33,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741885_1063 (size=694) 2024-11-21T12:30:33,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741885_1063 (size=694) 2024-11-21T12:30:33,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741885_1063 (size=694) 2024-11-21T12:30:33,175 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6 2024-11-21T12:30:33,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741886_1064 (size=77) 2024-11-21T12:30:33,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741886_1064 (size=77) 2024-11-21T12:30:33,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741886_1064 (size=77) 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:33,188 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,188 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,188 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192233188Disabling compacts and flushes for region at 1732192233188Disabling writes for close at 1732192233188Writing region close event to WAL at 1732192233188Closed at 1732192233188 2024-11-21T12:30:33,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T12:30:33,198 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732192233191"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732192233191"}]},"ts":"1732192233191"} 2024-11-21T12:30:33,202 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T12:30:33,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T12:30:33,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732192233204"}]},"ts":"1732192233204"} 2024-11-21T12:30:33,212 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-21T12:30:33,213 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9bb5b7e9dd2c=0} racks are {/default-rack=0} 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T12:30:33,214 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T12:30:33,214 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T12:30:33,214 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T12:30:33,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T12:30:33,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN}] 2024-11-21T12:30:33,218 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN 2024-11-21T12:30:33,220 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN; state=OFFLINE, location=9bb5b7e9dd2c,41613,1732192216530; forceNewPlan=false, retain=false 2024-11-21T12:30:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T12:30:33,373 INFO [9bb5b7e9dd2c:32899 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T12:30:33,374 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPENING, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:33,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN because future has completed 2024-11-21T12:30:33,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530}] 2024-11-21T12:30:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T12:30:33,539 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,539 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:33,540 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,540 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:33,540 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,540 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,542 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,544 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:33,544 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:33,544 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:33,545 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,546 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:33,546 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:33,547 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:33,547 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,548 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,548 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,549 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,549 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,551 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-21T12:30:33,552 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,557 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:33,558 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ca0f0d87cbf2161be78e638a1338fb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71694793, jitterRate=0.06833566725254059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-21T12:30:33,558 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,559 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ca0f0d87cbf2161be78e638a1338fb7: Running coprocessor pre-open hook at 1732192233540Writing region info on filesystem at 1732192233540Initializing all the Stores at 1732192233541 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192233541Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192233542 (+1 ms)Cleaning up temporary data from old regions at 1732192233549 (+7 ms)Running coprocessor post-open hooks at 1732192233558 (+9 ms)Region opened successfully at 1732192233558 2024-11-21T12:30:33,561 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., pid=6, masterSystemTime=1732192233533 2024-11-21T12:30:33,564 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,564 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPEN, openSeqNum=2, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:33,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 because future has completed 2024-11-21T12:30:33,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T12:30:33,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 in 192 msec 2024-11-21T12:30:33,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T12:30:33,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN in 358 msec 2024-11-21T12:30:33,579 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T12:30:33,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732192233580"}]},"ts":"1732192233580"} 2024-11-21T12:30:33,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-21T12:30:33,585 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T12:30:33,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 440 msec 2024-11-21T12:30:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T12:30:33,795 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-21T12:30:33,795 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-21T12:30:33,797 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T12:30:33,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-21T12:30:33,803 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T12:30:33,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-21T12:30:33,819 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=2] 2024-11-21T12:30:33,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6ca0f0d87cbf2161be78e638a1338fb7, source=9bb5b7e9dd2c,41613,1732192216530, destination=9bb5b7e9dd2c,36317,1732192216659, warming up region on 9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T12:30:33,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6ca0f0d87cbf2161be78e638a1338fb7, source=9bb5b7e9dd2c,41613,1732192216530, destination=9bb5b7e9dd2c,36317,1732192216659, running balancer 2024-11-21T12:30:33,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45343, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T12:30:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE 2024-11-21T12:30:33,844 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE 2024-11-21T12:30:33,846 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=CLOSING, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:33,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:33,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE because future has completed 2024-11-21T12:30:33,849 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,850 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T12:30:33,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530}] 2024-11-21T12:30:33,851 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:33,851 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:33,852 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:33,852 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:33,853 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:33,853 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:33,855 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:33,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192233856Disabling compacts and flushes for region at 1732192233856Disabling writes for close at 1732192233856Writing region close event to WAL at 1732192233861 (+5 ms)Closed at 1732192233861 2024-11-21T12:30:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T12:30:34,011 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,012 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T12:30:34,013 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:34,013 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,013 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,013 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:34,013 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,013 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 6ca0f0d87cbf2161be78e638a1338fb7 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-21T12:30:34,037 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/8e791bdd46c3454fa5a74686b17af52b is 35, key is r1/cf1:q/1732192233822/Put/seqid=0 2024-11-21T12:30:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741887_1065 (size=4783) 2024-11-21T12:30:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741887_1065 (size=4783) 2024-11-21T12:30:34,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741887_1065 (size=4783) 2024-11-21T12:30:34,047 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/8e791bdd46c3454fa5a74686b17af52b 2024-11-21T12:30:34,057 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/8e791bdd46c3454fa5a74686b17af52b as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b 2024-11-21T12:30:34,065 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b, entries=1, sequenceid=5, filesize=4.7 K 2024-11-21T12:30:34,067 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6ca0f0d87cbf2161be78e638a1338fb7 in 54ms, sequenceid=5, compaction requested=false 2024-11-21T12:30:34,067 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-21T12:30:34,074 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-21T12:30:34,078 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,078 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192234013Running coprocessor pre-close hooks at 1732192234013Disabling compacts and flushes for region at 1732192234013Disabling writes for close at 1732192234013Obtaining lock to block concurrent updates at 1732192234013Preparing flush snapshotting stores in 6ca0f0d87cbf2161be78e638a1338fb7 at 1732192234013Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732192234014 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. at 1732192234015 (+1 ms)Flushing 6ca0f0d87cbf2161be78e638a1338fb7/cf1: creating writer at 1732192234015Flushing 6ca0f0d87cbf2161be78e638a1338fb7/cf1: appending metadata at 1732192234037 (+22 ms)Flushing 6ca0f0d87cbf2161be78e638a1338fb7/cf1: closing flushed file at 1732192234037Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fb499a2: reopening flushed file at 1732192234055 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6ca0f0d87cbf2161be78e638a1338fb7 in 54ms, sequenceid=5, compaction requested=false at 1732192234067 (+12 ms)Writing region close event to WAL at 1732192234069 (+2 ms)Running coprocessor post-close hooks at 1732192234075 (+6 ms)Closed at 1732192234078 (+3 ms) 2024-11-21T12:30:34,079 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6ca0f0d87cbf2161be78e638a1338fb7 move to 9bb5b7e9dd2c,36317,1732192216659 record at close sequenceid=5 2024-11-21T12:30:34,083 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,084 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=CLOSED 2024-11-21T12:30:34,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 because future has completed 2024-11-21T12:30:34,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T12:30:34,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 in 239 msec 2024-11-21T12:30:34,094 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE; state=CLOSED, location=9bb5b7e9dd2c,36317,1732192216659; forceNewPlan=false, retain=false 2024-11-21T12:30:34,245 INFO [9bb5b7e9dd2c:32899 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T12:30:34,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPENING, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:34,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE because future has completed 2024-11-21T12:30:34,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659}] 2024-11-21T12:30:34,410 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,410 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:34,411 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,411 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:34,411 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,411 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,414 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,415 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:34,415 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:34,424 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b 2024-11-21T12:30:34,425 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:34,425 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,426 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:34,427 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:34,427 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:34,428 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,429 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,431 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,432 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,432 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,433 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-21T12:30:34,435 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,436 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6ca0f0d87cbf2161be78e638a1338fb7; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60190912, jitterRate=-0.10308551788330078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-21T12:30:34,437 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,438 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6ca0f0d87cbf2161be78e638a1338fb7: Running coprocessor pre-open hook at 1732192234412Writing region info on filesystem at 1732192234412Initializing all the Stores at 1732192234413 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192234413Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192234413Cleaning up temporary data from old regions at 1732192234432 (+19 ms)Running coprocessor post-open hooks at 1732192234437 (+5 ms)Region opened successfully at 1732192234437 2024-11-21T12:30:34,440 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., pid=9, masterSystemTime=1732192234404 2024-11-21T12:30:34,443 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,443 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,444 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPEN, openSeqNum=9, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:34,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 because future has completed 2024-11-21T12:30:34,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T12:30:34,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 in 202 msec 2024-11-21T12:30:34,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE in 614 msec 2024-11-21T12:30:34,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T12:30:34,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T12:30:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:32810 deadline: 1732192294508, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=36317 startCode=1732192216659. As of locationSeqNum=5. 2024-11-21T12:30:34,518 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=36317 startCode=1732192216659. As of locationSeqNum=5. 2024-11-21T12:30:34,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=36317 startCode=1732192216659. As of locationSeqNum=5. 2024-11-21T12:30:34,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=36317 startCode=1732192216659. As of locationSeqNum=5. 2024-11-21T12:30:34,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T12:30:34,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T12:30:34,648 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ca0f0d87cbf2161be78e638a1338fb7 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-21T12:30:34,676 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/4e8ea0ec107d484495227c779128b2ee is 29, key is r1/cf1:/1732192234638/DeleteFamily/seqid=0 2024-11-21T12:30:34,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741888_1066 (size=4906) 2024-11-21T12:30:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741888_1066 (size=4906) 2024-11-21T12:30:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741888_1066 (size=4906) 2024-11-21T12:30:34,688 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,695 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf2/a3f1b5b81a8d447795425b62c58858f9 is 29, key is r1/cf2:/1732192234638/DeleteFamily/seqid=0 2024-11-21T12:30:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741889_1067 (size=4906) 2024-11-21T12:30:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741889_1067 (size=4906) 2024-11-21T12:30:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741889_1067 (size=4906) 2024-11-21T12:30:34,725 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf2/a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,732 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,733 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/4e8ea0ec107d484495227c779128b2ee as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,742 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,742 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee, entries=1, sequenceid=12, filesize=4.8 K 2024-11-21T12:30:34,743 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf2/a3f1b5b81a8d447795425b62c58858f9 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,751 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,751 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9, entries=1, sequenceid=12, filesize=4.8 K 2024-11-21T12:30:34,753 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 6ca0f0d87cbf2161be78e638a1338fb7 in 105ms, sequenceid=12, compaction requested=false 2024-11-21T12:30:34,753 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6ca0f0d87cbf2161be78e638a1338fb7: 2024-11-21T12:30:34,756 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T12:30:34,758 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6ca0f0d87cbf2161be78e638a1338fb7/cf1 is initiating major compaction (all files) 2024-11-21T12:30:34,758 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T12:30:34,758 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:34,759 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6ca0f0d87cbf2161be78e638a1338fb7/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,759 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b, hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee] into tmpdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp, totalSize=9.5 K 2024-11-21T12:30:34,761 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8e791bdd46c3454fa5a74686b17af52b, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732192233822 2024-11-21T12:30:34,761 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4e8ea0ec107d484495227c779128b2ee, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-21T12:30:34,777 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6ca0f0d87cbf2161be78e638a1338fb7#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T12:30:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741890_1068 (size=4626) 2024-11-21T12:30:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741890_1068 (size=4626) 2024-11-21T12:30:34,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741890_1068 (size=4626) 2024-11-21T12:30:34,805 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf1/15588b6f8df440168c328c5b6b9f3990 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/15588b6f8df440168c328c5b6b9f3990 2024-11-21T12:30:34,821 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 6ca0f0d87cbf2161be78e638a1338fb7/cf1 of 6ca0f0d87cbf2161be78e638a1338fb7 into 15588b6f8df440168c328c5b6b9f3990(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T12:30:34,822 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6ca0f0d87cbf2161be78e638a1338fb7: 2024-11-21T12:30:34,822 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-21T12:30:34,822 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6ca0f0d87cbf2161be78e638a1338fb7/cf2 is initiating major compaction (all files) 2024-11-21T12:30:34,822 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T12:30:34,822 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T12:30:34,822 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6ca0f0d87cbf2161be78e638a1338fb7/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,822 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9] into tmpdir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp, totalSize=4.8 K 2024-11-21T12:30:34,823 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a3f1b5b81a8d447795425b62c58858f9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-21T12:30:34,829 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6ca0f0d87cbf2161be78e638a1338fb7#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T12:30:34,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741891_1069 (size=4592) 2024-11-21T12:30:34,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741891_1069 (size=4592) 2024-11-21T12:30:34,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741891_1069 (size=4592) 2024-11-21T12:30:34,848 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/.tmp/cf2/8aaf541cb3d244f1978376eb0304c003 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/8aaf541cb3d244f1978376eb0304c003 2024-11-21T12:30:34,858 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 6ca0f0d87cbf2161be78e638a1338fb7/cf2 of 6ca0f0d87cbf2161be78e638a1338fb7 into 8aaf541cb3d244f1978376eb0304c003(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T12:30:34,858 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6ca0f0d87cbf2161be78e638a1338fb7: 2024-11-21T12:30:34,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6ca0f0d87cbf2161be78e638a1338fb7, source=9bb5b7e9dd2c,36317,1732192216659, destination=9bb5b7e9dd2c,41613,1732192216530, warming up region on 9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:34,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6ca0f0d87cbf2161be78e638a1338fb7, source=9bb5b7e9dd2c,36317,1732192216659, destination=9bb5b7e9dd2c,41613,1732192216530, running balancer 2024-11-21T12:30:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE 2024-11-21T12:30:34,866 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE 2024-11-21T12:30:34,868 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=CLOSING, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:34,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:34,870 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,871 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:34,871 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:34,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE because future has completed 2024-11-21T12:30:34,872 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T12:30:34,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659}] 2024-11-21T12:30:34,882 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/15588b6f8df440168c328c5b6b9f3990 2024-11-21T12:30:34,888 INFO [StoreFileOpener-6ca0f0d87cbf2161be78e638a1338fb7-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,888 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:34,894 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b 2024-11-21T12:30:34,894 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:34,895 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:34,896 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:34,896 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:34,905 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/8aaf541cb3d244f1978376eb0304c003 2024-11-21T12:30:34,911 INFO [StoreFileOpener-6ca0f0d87cbf2161be78e638a1338fb7-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,911 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:34,911 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:34,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192234912Disabling compacts and flushes for region at 1732192234912Disabling writes for close at 1732192234912Writing region close event to WAL at 1732192234913 (+1 ms)Closed at 1732192234913 2024-11-21T12:30:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-21T12:30:35,027 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,027 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T12:30:35,027 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:35,027 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,027 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,027 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:35,027 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,028 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b, hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee] to archive 2024-11-21T12:30:35,032 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T12:30:35,036 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/8e791bdd46c3454fa5a74686b17af52b 2024-11-21T12:30:35,038 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/4e8ea0ec107d484495227c779128b2ee 2024-11-21T12:30:35,053 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9] to archive 2024-11-21T12:30:35,054 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T12:30:35,056 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9 to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/a3f1b5b81a8d447795425b62c58858f9 2024-11-21T12:30:35,062 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-21T12:30:35,063 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,063 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192235027Running coprocessor pre-close hooks at 1732192235027Disabling compacts and flushes for region at 1732192235027Disabling writes for close at 1732192235027Writing region close event to WAL at 1732192235058 (+31 ms)Running coprocessor post-close hooks at 1732192235062 (+4 ms)Closed at 1732192235063 (+1 ms) 2024-11-21T12:30:35,063 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 6ca0f0d87cbf2161be78e638a1338fb7 move to 9bb5b7e9dd2c,41613,1732192216530 record at close sequenceid=12 2024-11-21T12:30:35,066 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,068 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=CLOSED 2024-11-21T12:30:35,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 because future has completed 2024-11-21T12:30:35,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T12:30:35,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 in 200 msec 2024-11-21T12:30:35,077 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE; state=CLOSED, location=9bb5b7e9dd2c,41613,1732192216530; forceNewPlan=false, retain=false 2024-11-21T12:30:35,227 INFO [9bb5b7e9dd2c:32899 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T12:30:35,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPENING, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:35,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE because future has completed 2024-11-21T12:30:35,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530}] 2024-11-21T12:30:35,390 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,390 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:35,391 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,391 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:35,391 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,391 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,393 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,394 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:35,394 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:35,403 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/15588b6f8df440168c328c5b6b9f3990 2024-11-21T12:30:35,404 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:35,404 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,405 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:35,405 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:35,414 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/8aaf541cb3d244f1978376eb0304c003 2024-11-21T12:30:35,415 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:35,415 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,416 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,418 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,419 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,419 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,420 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-21T12:30:35,422 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,423 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 6ca0f0d87cbf2161be78e638a1338fb7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72265887, jitterRate=0.07684563100337982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-21T12:30:35,423 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,424 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6ca0f0d87cbf2161be78e638a1338fb7: Running coprocessor pre-open hook at 1732192235391Writing region info on filesystem at 1732192235391Initializing all the Stores at 1732192235392 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192235392Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192235392Cleaning up temporary data from old regions at 1732192235419 (+27 ms)Running coprocessor post-open hooks at 1732192235423 (+4 ms)Region opened successfully at 1732192235424 (+1 ms) 2024-11-21T12:30:35,426 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., pid=12, masterSystemTime=1732192235384 2024-11-21T12:30:35,428 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,429 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPEN, openSeqNum=18, regionLocation=9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:35,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 because future has completed 2024-11-21T12:30:35,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-21T12:30:35,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,41613,1732192216530 in 203 msec 2024-11-21T12:30:35,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, REOPEN/MOVE in 573 msec 2024-11-21T12:30:35,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T12:30:35,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T12:30:35,473 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9bb5b7e9dd2c,41613,1732192216530: testing ***** 2024-11-21T12:30:35,473 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T12:30:35,475 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T12:30:35,477 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T12:30:35,483 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T12:30:35,485 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T12:30:35,500 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 473105912 }, "NonHeapMemoryUsage": { "committed": 172359680, "init": 7667712, "max": -1, "used": 169784160 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9bb5b7e9dd2c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 48, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 17, "ProcessCallTime_98th_percentile": 48, "ProcessCallTime_99th_percentile": 48, "ProcessCallTime_99.9th_percentile": 48, "ProcessCallTime_TimeRangeCount_0-1": 8, "ProcessCallTime_TimeRangeCount_1-3": 16, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 48, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 18, "TotalCallTime_98th_percentile": 48, "TotalCallTime_99th_percentile": 48, "TotalCallTime_99.9th_percentile": 48, "TotalCallTime_TimeRangeCount_0-1": 8, "TotalCallTime_TimeRangeCount_1-3": 15, "TotalCallTime_TimeRangeCount_3-10": 6, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9bb5b7e9dd2c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55541", "tag.serverName": "9bb5b7e9dd2c,36317,1732192216659", "tag.clusterId": "0ec7614b-bbbd-45b9-88cc-598d55c1d096", "tag.Context": "regionserver", "tag.Hostname": "9bb5b7e9dd2c", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732192216659, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 35, "MajorCompactionTime_max": 60, "MajorCompactionTime_mean": 47, "MajorCompactionTime_25th_percentile": 35, "MajorCompactionTime_median": 36, "MajorCompactionTime_75th_percentile": 60, "MajorCompactionTime_90th_percentile": 60, "MajorCompactionTime_95th_percentile": 60, "MajorCompactionTime_98th_percentile": 60, "MajorCompactionTime_99th_percentile": 60, "MajorCompactionTime_99.9th_percentile": 60, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 3, "Delete_max": 3, "Delete_mean": 3, "Delete_25th_percentile": 3, "Delete_median": 3, "Delete_75th_percentile": 3, "Delete_90th_percentile": 3, "Delete_95th_percentile": 3, "Delete_98th_percentile": 3, "Delete_99th_percentile": 3, "Delete_99.9th_percentile": 3, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 35, "CompactionTime_max": 60, "CompactionTime_mean": 47, "CompactionTime_25th_percentile": 35, "CompactionTime_median": 36, "CompactionTime_75th_percentile": 60, "CompactionTime_90th_percentile": 60, "CompactionTime_95th_percentile": 60, "CompactionTime_98th_percentile": 60, "CompactionTime_99th_percentile": 60, "CompactionTime_99.9th_percentile": 60, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 105, "FlushTime_max": 105, "FlushTime_mean": 105, "FlushTime_25th_percentile": 105, "FlushTime_median": 105, "FlushTime_75th_percentile": 105, "FlushTime_90th_percentile": 105, "FlushTime_95th_percentile": 105, "FlushTime_98th_percentile": 105, "FlushTime_99th_percentile": 105, "FlushTime_99.9th_percentile": 105, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T12:30:35,505 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] master.MasterRpcServices(700): 9bb5b7e9dd2c,41613,1732192216530 reported a fatal error: ***** ABORTING region server 9bb5b7e9dd2c,41613,1732192216530: testing ***** 2024-11-21T12:30:35,507 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9bb5b7e9dd2c,41613,1732192216530' ***** 2024-11-21T12:30:35,507 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-21T12:30:35,508 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T12:30:35,508 INFO [RS:0;9bb5b7e9dd2c:41613 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-21T12:30:35,508 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T12:30:35,508 INFO [RS:0;9bb5b7e9dd2c:41613 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-21T12:30:35,509 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,509 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(956): aborting server 9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:35,509 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T12:30:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36317 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:43144 deadline: 1732192295509, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=41613 startCode=1732192216530. As of locationSeqNum=12. 2024-11-21T12:30:35,510 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:35,510 INFO [RS:0;9bb5b7e9dd2c:41613 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9bb5b7e9dd2c:41613. 2024-11-21T12:30:35,510 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,510 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,510 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T12:30:35,510 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:35,510 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,510 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=41613 startCode=1732192216530. As of locationSeqNum=12. 2024-11-21T12:30:35,510 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=41613 startCode=1732192216530. As of locationSeqNum=12. 2024-11-21T12:30:35,510 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:35,510 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9bb5b7e9dd2c port=41613 startCode=1732192216530. As of locationSeqNum=12. 2024-11-21T12:30:35,511 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T12:30:35,511 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T12:30:35,511 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T12:30:35,511 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T12:30:35,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:35,514 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T12:30:35,514 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6ca0f0d87cbf2161be78e638a1338fb7=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.} 2024-11-21T12:30:35,514 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T12:30:35,514 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T12:30:35,514 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T12:30:35,514 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T12:30:35,514 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:35,514 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T12:30:35,514 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting 2024-11-21T12:30:35,514 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting 2024-11-21T12:30:35,515 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 from cache 2024-11-21T12:30:35,517 ERROR [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-21T12:30:35,517 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T12:30:35,517 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,517 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T12:30:35,517 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192235509Running coprocessor pre-close hooks at 1732192235509Disabling compacts and flushes for region at 1732192235509Disabling writes for close at 1732192235510 (+1 ms)Writing region close event to WAL at 1732192235517 (+7 ms)Running coprocessor post-close hooks at 1732192235517Closed at 1732192235517 2024-11-21T12:30:35,517 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732192235514Running coprocessor pre-close hooks at 1732192235514Disabling compacts and flushes for region at 1732192235514Disabling writes for close at 1732192235514Writing region close event to WAL at 1732192235517 (+3 ms)Running coprocessor post-close hooks at 1732192235517Closed at 1732192235517 2024-11-21T12:30:35,518 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:35,518 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T12:30:35,575 INFO [regionserver/9bb5b7e9dd2c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:35,623 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:35,624 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1] 2024-11-21T12:30:35,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:35,626 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting 2024-11-21T12:30:35,626 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9bb5b7e9dd2c,41613,1732192216530 aborting 2024-11-21T12:30:35,626 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 from cache 2024-11-21T12:30:35,715 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(976): stopping server 9bb5b7e9dd2c,41613,1732192216530; all regions closed. 2024-11-21T12:30:35,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741836_1012 (size=7323) 2024-11-21T12:30:35,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741836_1012 (size=7323) 2024-11-21T12:30:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741836_1012 (size=7323) 2024-11-21T12:30:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741834_1010 (size=2288) 2024-11-21T12:30:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741834_1010 (size=2288) 2024-11-21T12:30:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741834_1010 (size=2288) 2024-11-21T12:30:35,724 DEBUG [RS:0;9bb5b7e9dd2c:41613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:35,724 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:35,724 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T12:30:35,724 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.ChoreService(370): Chore service for: regionserver/9bb5b7e9dd2c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T12:30:35,724 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T12:30:35,724 INFO [regionserver/9bb5b7e9dd2c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T12:30:35,725 INFO [RS:0;9bb5b7e9dd2c:41613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41613 2024-11-21T12:30:35,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T12:30:35,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9bb5b7e9dd2c,41613,1732192216530 2024-11-21T12:30:35,733 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T12:30:35,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9bb5b7e9dd2c,41613,1732192216530] 2024-11-21T12:30:35,736 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9bb5b7e9dd2c,41613,1732192216530 already deleted, retry=false 2024-11-21T12:30:35,737 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9bb5b7e9dd2c,41613,1732192216530 on 9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:35,742 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9bb5b7e9dd2c,41613,1732192216530, splitWal=true, meta=true 2024-11-21T12:30:35,745 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 9bb5b7e9dd2c,41613,1732192216530 (carryingMeta=true) 9bb5b7e9dd2c,41613,1732192216530/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@72ec67c5[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T12:30:35,746 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9bb5b7e9dd2c,41613,1732192216530, splitWal=true, meta=true 2024-11-21T12:30:35,747 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9bb5b7e9dd2c,41613,1732192216530, splitWal=true, meta=true, isMeta: true 2024-11-21T12:30:35,749 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting 2024-11-21T12:30:35,751 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9bb5b7e9dd2c,41613,1732192216530 WAL count=1, meta=true 2024-11-21T12:30:35,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta}] 2024-11-21T12:30:35,763 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:35,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, worker=9bb5b7e9dd2c,44681,1732192216704}] 2024-11-21T12:30:35,833 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:35,835 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1] 2024-11-21T12:30:35,837 INFO [RS:0;9bb5b7e9dd2c:41613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T12:30:35,837 INFO [RS:0;9bb5b7e9dd2c:41613 {}] regionserver.HRegionServer(1031): Exiting; stopping=9bb5b7e9dd2c,41613,1732192216530; zookeeper connection closed. 2024-11-21T12:30:35,837 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9bb5b7e9dd2c:41613 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:35,838 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1, error=java.net.ConnectException: Call to address=9bb5b7e9dd2c:41613 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 2024-11-21T12:30:35,838 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-21T12:30:35,838 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 from cache 2024-11-21T12:30:35,839 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 9bb5b7e9dd2c:41613 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 2024-11-21T12:30:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:35,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41613-0x1006251a9c40001, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:35,845 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11aab86a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11aab86a 2024-11-21T12:30:35,931 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T12:30:35,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60133, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T12:30:35,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44681 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-21T12:30:35,953 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, size=7.2 K (7323bytes) 2024-11-21T12:30:35,954 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta 2024-11-21T12:30:35,954 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta after 0ms 2024-11-21T12:30:35,985 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:35,985 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta took 32ms 2024-11-21T12:30:35,994 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-21T12:30:35,995 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta so closing down 2024-11-21T12:30:35,995 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:35,996 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp 2024-11-21T12:30:36,026 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp 2024-11-21T12:30:36,026 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741892_1070 (size=7071) 2024-11-21T12:30:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741892_1070 (size=7071) 2024-11-21T12:30:36,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741892_1070 (size=7071) 2024-11-21T12:30:36,038 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:36,039 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-21T12:30:36,040 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 53 ms; skipped=1; WAL=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-21T12:30:36,040 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, journal: Splitting hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, size=7.2 K (7323bytes) at 1732192235954Finishing writing output for hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta so closing down at 1732192235995 (+41 ms)Creating recovered edits writer path=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp at 1732192236026 (+31 ms)3 split writer threads finished at 1732192236027 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732192236038 (+11 ms)Rename recovered edits hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta.temp to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732192236039 (+1 ms)Processed 16 edits across 1 Regions in 53 ms; skipped=1; WAL=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1732192236040 (+1 ms) 2024-11-21T12:30:36,040 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta 2024-11-21T12:30:36,042 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-21T12:30:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-21T12:30:36,050 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs 2024-11-21T12:30:36,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-21T12:30:36,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, worker=9bb5b7e9dd2c,44681,1732192216704 in 284 msec 2024-11-21T12:30:36,056 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:36,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T12:30:36,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.meta.1732192218816.meta, worker=9bb5b7e9dd2c,44681,1732192216704 in 303 msec 2024-11-21T12:30:36,066 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting dir is empty, no logs to split. 2024-11-21T12:30:36,066 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9bb5b7e9dd2c,41613,1732192216530 WAL count=0, meta=true 2024-11-21T12:30:36,066 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 9bb5b7e9dd2c,41613,1732192216530 WAL splitting is done? wals=0, meta=true 2024-11-21T12:30:36,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T12:30:36,071 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T12:30:36,072 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T12:30:36,143 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:36,145 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1] 2024-11-21T12:30:36,145 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9bb5b7e9dd2c:41613 this server is in the failed servers list 2024-11-21T12:30:36,146 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9bb5b7e9dd2c:41613 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,146 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,146 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 from cache 2024-11-21T12:30:36,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-21T12:30:36,207 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:36,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-21T12:30:36,209 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:36,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T12:30:36,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T12:30:36,222 DEBUG [9bb5b7e9dd2c:32899 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(204): Hosts are {9bb5b7e9dd2c=0} racks are {/default-rack=0} 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T12:30:36,223 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T12:30:36,223 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T12:30:36,223 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T12:30:36,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:36,225 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9bb5b7e9dd2c,36317,1732192216659, state=OPENING 2024-11-21T12:30:36,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,228 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T12:30:36,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,36317,1732192216659}] 2024-11-21T12:30:36,386 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T12:30:36,386 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:36,386 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T12:30:36,388 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9bb5b7e9dd2c%2C36317%2C1732192216659.meta, suffix=.meta, logDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659, archiveDir=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs, maxLogs=32 2024-11-21T12:30:36,403 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659/9bb5b7e9dd2c%2C36317%2C1732192216659.meta.1732192236389.meta, exclude list is [], retry=0 2024-11-21T12:30:36,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:36,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:36,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:36,437 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,36317,1732192216659/9bb5b7e9dd2c%2C36317%2C1732192216659.meta.1732192236389.meta 2024-11-21T12:30:36,437 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T12:30:36,438 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T12:30:36,438 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T12:30:36,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T12:30:36,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T12:30:36,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:36,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:36,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T12:30:36,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T12:30:36,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:36,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:36,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T12:30:36,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T12:30:36,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:36,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:36,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T12:30:36,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T12:30:36,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:36,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T12:30:36,447 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T12:30:36,448 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:36,449 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740 2024-11-21T12:30:36,450 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-21T12:30:36,480 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:36,483 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-21T12:30:36,483 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-21T12:30:36,506 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/b3bd92f05e434f36a438025ea6510459 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7./info:regioninfo/1732192235429/Put/seqid=0 2024-11-21T12:30:36,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741894_1072 (size=11177) 2024-11-21T12:30:36,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741894_1072 (size=11177) 2024-11-21T12:30:36,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741894_1072 (size=11177) 2024-11-21T12:30:36,514 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/b3bd92f05e434f36a438025ea6510459 2024-11-21T12:30:36,537 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/ns/141389f66ebe4c69b5281f046870a7f8 is 43, key is default/ns:d/1732192219074/Put/seqid=0 2024-11-21T12:30:36,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741895_1073 (size=5153) 2024-11-21T12:30:36,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741895_1073 (size=5153) 2024-11-21T12:30:36,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741895_1073 (size=5153) 2024-11-21T12:30:36,545 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/ns/141389f66ebe4c69b5281f046870a7f8 2024-11-21T12:30:36,567 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/table/346f3536b06146f0a742ff35466b011d is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732192233580/Put/seqid=0 2024-11-21T12:30:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741896_1074 (size=5431) 2024-11-21T12:30:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741896_1074 (size=5431) 2024-11-21T12:30:36,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741896_1074 (size=5431) 2024-11-21T12:30:36,576 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/table/346f3536b06146f0a742ff35466b011d 2024-11-21T12:30:36,583 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/b3bd92f05e434f36a438025ea6510459 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/info/b3bd92f05e434f36a438025ea6510459 2024-11-21T12:30:36,590 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/info/b3bd92f05e434f36a438025ea6510459, entries=36, sequenceid=18, filesize=10.9 K 2024-11-21T12:30:36,591 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/ns/141389f66ebe4c69b5281f046870a7f8 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/ns/141389f66ebe4c69b5281f046870a7f8 2024-11-21T12:30:36,597 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/ns/141389f66ebe4c69b5281f046870a7f8, entries=2, sequenceid=18, filesize=5.0 K 2024-11-21T12:30:36,599 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/table/346f3536b06146f0a742ff35466b011d as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/table/346f3536b06146f0a742ff35466b011d 2024-11-21T12:30:36,605 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/table/346f3536b06146f0a742ff35466b011d, entries=2, sequenceid=18, filesize=5.3 K 2024-11-21T12:30:36,606 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 123ms, sequenceid=18, compaction requested=false; wal=null 2024-11-21T12:30:36,606 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T12:30:36,607 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-21T12:30:36,612 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T12:30:36,613 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T12:30:36,614 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T12:30:36,615 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T12:30:36,618 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T12:30:36,619 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64519860, jitterRate=-0.03857916593551636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T12:30:36,620 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T12:30:36,621 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732192236439Writing region info on filesystem at 1732192236439Initializing all the Stores at 1732192236440 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192236440Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192236440Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192236440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732192236440Obtaining lock to block concurrent updates at 1732192236483 (+43 ms)Preparing flush snapshotting stores in 1588230740 at 1732192236483Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732192236483Flushing stores of hbase:meta,,1.1588230740 at 1732192236483Flushing 1588230740/info: creating writer at 1732192236484 (+1 ms)Flushing 1588230740/info: appending metadata at 1732192236505 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732192236505Flushing 1588230740/ns: creating writer at 1732192236521 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732192236536 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732192236536Flushing 1588230740/table: creating writer at 1732192236551 (+15 ms)Flushing 1588230740/table: appending metadata at 1732192236566 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732192236566Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cd6f479: reopening flushed file at 1732192236582 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acdccd7: reopening flushed file at 1732192236590 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5db43c56: reopening flushed file at 1732192236598 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 123ms, sequenceid=18, compaction requested=false; wal=null at 1732192236606 (+8 ms)Cleaning up temporary data from old regions at 1732192236613 (+7 ms)Running coprocessor post-open hooks at 1732192236620 (+7 ms)Region opened successfully at 1732192236621 (+1 ms) 2024-11-21T12:30:36,622 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732192236381 2024-11-21T12:30:36,625 DEBUG [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T12:30:36,625 INFO [RS_OPEN_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T12:30:36,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:36,627 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9bb5b7e9dd2c,36317,1732192216659, state=OPEN 2024-11-21T12:30:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T12:30:36,629 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:36,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T12:30:36,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-21T12:30:36,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9bb5b7e9dd2c,36317,1732192216659 in 401 msec 2024-11-21T12:30:36,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-21T12:30:36,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 566 msec 2024-11-21T12:30:36,640 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9bb5b7e9dd2c,41613,1732192216530 had 2 regions 2024-11-21T12:30:36,642 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9bb5b7e9dd2c,41613,1732192216530, splitWal=true, meta=true, isMeta: false 2024-11-21T12:30:36,644 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9bb5b7e9dd2c,41613,1732192216530 WAL count=1, meta=false 2024-11-21T12:30:36,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415}] 2024-11-21T12:30:36,648 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:36,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, worker=9bb5b7e9dd2c,36317,1732192216659}] 2024-11-21T12:30:36,653 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:36,654 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=-1] 2024-11-21T12:30:36,773 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9bb5b7e9dd2c:41613 this server is in the failed servers list 2024-11-21T12:30:36,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9bb5b7e9dd2c:41613 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,774 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=12 from cache 2024-11-21T12:30:36,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-21T12:30:36,820 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, size=2.2 K (2288bytes) 2024-11-21T12:30:36,820 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 2024-11-21T12:30:36,821 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 after 1ms 2024-11-21T12:30:36,851 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:36,851 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 took 32ms 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 6ca0f0d87cbf2161be78e638a1338fb7: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 so closing down 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:36,855 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:36,855 INFO [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, journal: Splitting hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, size=2.2 K (2288bytes) at 1732192236820Finishing writing output for hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 so closing down at 1732192236855 (+35 ms)3 split writer threads finished at 1732192236855Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732192236855 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 2024-11-21T12:30:36,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9bb5b7e9dd2c:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-21T12:30:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-21T12:30:36,860 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting/9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415 to hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs 2024-11-21T12:30:36,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-21T12:30:36,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, worker=9bb5b7e9dd2c,36317,1732192216659 in 211 msec 2024-11-21T12:30:36,865 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:36,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-21T12:30:36,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9bb5b7e9dd2c%2C41613%2C1732192216530.1732192218415, worker=9bb5b7e9dd2c,36317,1732192216659 in 220 msec 2024-11-21T12:30:36,869 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/WALs/9bb5b7e9dd2c,41613,1732192216530-splitting dir is empty, no logs to split. 2024-11-21T12:30:36,869 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9bb5b7e9dd2c,41613,1732192216530 WAL count=0, meta=false 2024-11-21T12:30:36,869 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9bb5b7e9dd2c,41613,1732192216530 WAL splitting is done? wals=0, meta=false 2024-11-21T12:30:36,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN}] 2024-11-21T12:30:36,873 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN 2024-11-21T12:30:36,874 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T12:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18] 2024-11-21T12:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9bb5b7e9dd2c:41613 this server is in the failed servers list 2024-11-21T12:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9bb5b7e9dd2c:41613 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 from cache 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(204): Hosts are {9bb5b7e9dd2c=0} racks are {/default-rack=0} 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T12:30:37,025 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T12:30:37,025 INFO [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T12:30:37,025 DEBUG [9bb5b7e9dd2c:32899 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T12:30:37,026 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPENING, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:37,027 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9bb5b7e9dd2c:41613 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:37,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1, error=java.net.ConnectException: Call to address=9bb5b7e9dd2c:41613 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 2024-11-21T12:30:37,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T12:30:37,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=-1 from cache 2024-11-21T12:30:37,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9bb5b7e9dd2c:41613 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9bb5b7e9dd2c/172.17.0.2:41613 2024-11-21T12:30:37,133 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T12:30:37,133 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=-1] 2024-11-21T12:30:37,133 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T12:30:37,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T12:30:37,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN because future has completed 2024-11-21T12:30:37,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659}] 2024-11-21T12:30:37,294 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:37,294 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ca0f0d87cbf2161be78e638a1338fb7, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:37,295 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,295 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:37,295 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,295 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,297 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,297 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18] 2024-11-21T12:30:37,297 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9bb5b7e9dd2c:41613 this server is in the failed servers list 2024-11-21T12:30:37,298 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9bb5b7e9dd2c:41613 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:37,298 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9bb5b7e9dd2c:41613 2024-11-21T12:30:37,298 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,41613,1732192216530, seqNum=18 from cache 2024-11-21T12:30:37,298 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf1 2024-11-21T12:30:37,298 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:37,304 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf1/15588b6f8df440168c328c5b6b9f3990 2024-11-21T12:30:37,304 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:37,305 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,306 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ca0f0d87cbf2161be78e638a1338fb7 columnFamilyName cf2 2024-11-21T12:30:37,306 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:37,312 DEBUG [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/cf2/8aaf541cb3d244f1978376eb0304c003 2024-11-21T12:30:37,312 INFO [StoreOpener-6ca0f0d87cbf2161be78e638a1338fb7-1 {}] regionserver.HStore(327): Store=6ca0f0d87cbf2161be78e638a1338fb7/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:37,312 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,313 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,314 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,315 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,315 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,316 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-21T12:30:37,317 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,318 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 6ca0f0d87cbf2161be78e638a1338fb7; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66005819, jitterRate=-0.016436651349067688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-21T12:30:37,318 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:37,319 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 6ca0f0d87cbf2161be78e638a1338fb7: Running coprocessor pre-open hook at 1732192237295Writing region info on filesystem at 1732192237295Initializing all the Stores at 1732192237296 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192237296Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192237297 (+1 ms)Cleaning up temporary data from old regions at 1732192237315 (+18 ms)Running coprocessor post-open hooks at 1732192237318 (+3 ms)Region opened successfully at 1732192237319 (+1 ms) 2024-11-21T12:30:37,320 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., pid=21, masterSystemTime=1732192237290 2024-11-21T12:30:37,323 DEBUG [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:37,323 INFO [RS_OPEN_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:37,324 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6ca0f0d87cbf2161be78e638a1338fb7, regionState=OPEN, openSeqNum=18, regionLocation=9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:37,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 because future has completed 2024-11-21T12:30:37,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-21T12:30:37,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ca0f0d87cbf2161be78e638a1338fb7, server=9bb5b7e9dd2c,36317,1732192216659 in 190 msec 2024-11-21T12:30:37,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-21T12:30:37,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ca0f0d87cbf2161be78e638a1338fb7, ASSIGN in 458 msec 2024-11-21T12:30:37,334 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 9bb5b7e9dd2c,41613,1732192216530 after splitting done 2024-11-21T12:30:37,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9bb5b7e9dd2c,41613,1732192216530, splitWal=true, meta=true in 1.5960 sec 2024-11-21T12:30:37,807 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7., hostname=9bb5b7e9dd2c,36317,1732192216659, seqNum=18] 2024-11-21T12:30:37,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 416), OpenFileDescriptor=1046 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 172) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7021 (was 7097) 2024-11-21T12:30:37,825 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-21T12:30:37,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1046, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7021 2024-11-21T12:30:37,839 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-21T12:30:37,857 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:37,860 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:37,861 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:37,864 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81241158, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-81241158, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:37,879 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81241158/hregion-81241158.1732192237864, exclude list is [], retry=0 2024-11-21T12:30:37,883 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:37,883 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:37,884 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:37,914 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81241158/hregion-81241158.1732192237864 2024-11-21T12:30:37,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:37,915 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7fdb275382ff63d07a2c56d1ef8ba756, NAME => 'testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:37,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741898_1076 (size=67) 2024-11-21T12:30:37,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741898_1076 (size=67) 2024-11-21T12:30:37,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741898_1076 (size=67) 2024-11-21T12:30:37,926 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:37,927 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,929 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName a 2024-11-21T12:30:37,929 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:37,930 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:37,930 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,931 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName b 2024-11-21T12:30:37,931 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:37,932 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:37,932 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,933 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName c 2024-11-21T12:30:37,934 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:37,934 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:37,934 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,936 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,937 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:37,938 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:37,940 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:37,941 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7fdb275382ff63d07a2c56d1ef8ba756; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61430846, jitterRate=-0.08460906147956848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:37,941 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7fdb275382ff63d07a2c56d1ef8ba756: Writing region info on filesystem at 1732192237926Initializing all the Stores at 1732192237927 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192237927Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192237927Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192237927Cleaning up temporary data from old regions at 1732192237936 (+9 ms)Region opened successfully at 1732192237941 (+5 ms) 2024-11-21T12:30:37,941 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7fdb275382ff63d07a2c56d1ef8ba756, disabling compactions & flushes 2024-11-21T12:30:37,941 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:37,942 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:37,942 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. after waiting 0 ms 2024-11-21T12:30:37,942 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:37,942 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:37,942 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7fdb275382ff63d07a2c56d1ef8ba756: Waiting for close lock at 1732192237941Disabling compacts and flushes for region at 1732192237941Disabling writes for close at 1732192237942 (+1 ms)Writing region close event to WAL at 1732192237942Closed at 1732192237942 2024-11-21T12:30:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741897_1075 (size=175) 2024-11-21T12:30:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741897_1075 (size=175) 2024-11-21T12:30:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741897_1075 (size=175) 2024-11-21T12:30:37,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:37,948 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81241158:(num 1732192237864) 2024-11-21T12:30:37,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:37,949 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:37,964 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, exclude list is [], retry=0 2024-11-21T12:30:37,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:37,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:37,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:37,999 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 2024-11-21T12:30:37,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:37,999 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fdb275382ff63d07a2c56d1ef8ba756, NAME => 'testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:38,000 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:38,000 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,000 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,001 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,002 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName a 2024-11-21T12:30:38,003 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,003 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,003 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,004 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName b 2024-11-21T12:30:38,004 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,005 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,005 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,006 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName c 2024-11-21T12:30:38,006 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,007 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,007 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,011 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:38,012 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,013 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7fdb275382ff63d07a2c56d1ef8ba756; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62300270, jitterRate=-0.07165363430976868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:38,014 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7fdb275382ff63d07a2c56d1ef8ba756: Writing region info on filesystem at 1732192238000Initializing all the Stores at 1732192238001 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238001Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238001Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238001Cleaning up temporary data from old regions at 1732192238010 (+9 ms)Region opened successfully at 1732192238014 (+4 ms) 2024-11-21T12:30:38,045 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7fdb275382ff63d07a2c56d1ef8ba756 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-21T12:30:38,074 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/a/812dd7b250c94d55bfcef29ee753c6e0 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732192238014/Put/seqid=0 2024-11-21T12:30:38,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741900_1078 (size=5958) 2024-11-21T12:30:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741900_1078 (size=5958) 2024-11-21T12:30:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741900_1078 (size=5958) 2024-11-21T12:30:38,084 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/a/812dd7b250c94d55bfcef29ee753c6e0 2024-11-21T12:30:38,107 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/e5f17c10b72c4d4083a786654b3c7d48 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732192238025/Put/seqid=0 2024-11-21T12:30:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741901_1079 (size=5958) 2024-11-21T12:30:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741901_1079 (size=5958) 2024-11-21T12:30:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741901_1079 (size=5958) 2024-11-21T12:30:38,117 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/e5f17c10b72c4d4083a786654b3c7d48 2024-11-21T12:30:38,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/c/168b4c7f06b54c76a537f7b096f2e4b9 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732192238032/Put/seqid=0 2024-11-21T12:30:38,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741902_1080 (size=5958) 2024-11-21T12:30:38,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741902_1080 (size=5958) 2024-11-21T12:30:38,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741902_1080 (size=5958) 2024-11-21T12:30:38,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/c/168b4c7f06b54c76a537f7b096f2e4b9 2024-11-21T12:30:38,161 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/a/812dd7b250c94d55bfcef29ee753c6e0 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/a/812dd7b250c94d55bfcef29ee753c6e0 2024-11-21T12:30:38,169 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/a/812dd7b250c94d55bfcef29ee753c6e0, entries=10, sequenceid=33, filesize=5.8 K 2024-11-21T12:30:38,171 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/e5f17c10b72c4d4083a786654b3c7d48 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/e5f17c10b72c4d4083a786654b3c7d48 2024-11-21T12:30:38,178 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/e5f17c10b72c4d4083a786654b3c7d48, entries=10, sequenceid=33, filesize=5.8 K 2024-11-21T12:30:38,179 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/c/168b4c7f06b54c76a537f7b096f2e4b9 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/c/168b4c7f06b54c76a537f7b096f2e4b9 2024-11-21T12:30:38,187 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/c/168b4c7f06b54c76a537f7b096f2e4b9, entries=10, sequenceid=33, filesize=5.8 K 2024-11-21T12:30:38,189 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7fdb275382ff63d07a2c56d1ef8ba756 in 144ms, sequenceid=33, compaction requested=false 2024-11-21T12:30:38,189 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7fdb275382ff63d07a2c56d1ef8ba756: 2024-11-21T12:30:38,189 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7fdb275382ff63d07a2c56d1ef8ba756, disabling compactions & flushes 2024-11-21T12:30:38,189 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:38,189 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:38,189 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. after waiting 0 ms 2024-11-21T12:30:38,189 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:38,191 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. 2024-11-21T12:30:38,191 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7fdb275382ff63d07a2c56d1ef8ba756: Waiting for close lock at 1732192238189Disabling compacts and flushes for region at 1732192238189Disabling writes for close at 1732192238189Writing region close event to WAL at 1732192238191 (+2 ms)Closed at 1732192238191 2024-11-21T12:30:38,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741899_1077 (size=6311) 2024-11-21T12:30:38,195 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 not finished, retry = 0 2024-11-21T12:30:38,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741899_1077 (size=6311) 2024-11-21T12:30:38,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741899_1077 (size=6311) 2024-11-21T12:30:38,302 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/e5f17c10b72c4d4083a786654b3c7d48 to hdfs://localhost:41319/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/e5f17c10b72c4d4083a786654b3c7d48 2024-11-21T12:30:38,327 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, size=6.2 K (6311bytes) 2024-11-21T12:30:38,327 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 2024-11-21T12:30:38,328 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 after 1ms 2024-11-21T12:30:38,360 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:38,360 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 took 33ms 2024-11-21T12:30:38,363 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 so closing down 2024-11-21T12:30:38,363 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:38,365 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732192237950.temp 2024-11-21T12:30:38,395 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp 2024-11-21T12:30:38,396 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:38,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741903_1081 (size=5575) 2024-11-21T12:30:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741903_1081 (size=5575) 2024-11-21T12:30:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741903_1081 (size=5575) 2024-11-21T12:30:38,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:38,407 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 2024-11-21T12:30:38,407 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-21T12:30:38,407 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, journal: Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, size=6.2 K (6311bytes) at 1732192238327Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 so closing down at 1732192238363 (+36 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp at 1732192238395 (+32 ms)3 split writer threads finished at 1732192238396 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732192238405 (+9 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000003-wal.1732192237950.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 at 1732192238407 (+2 ms)Processed 32 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732192238407 2024-11-21T12:30:38,409 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192237950 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192237950 2024-11-21T12:30:38,410 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 2024-11-21T12:30:38,410 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:38,413 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:38,433 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192238413, exclude list is [], retry=0 2024-11-21T12:30:38,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:38,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:38,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:38,468 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192238413 2024-11-21T12:30:38,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:38,468 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fdb275382ff63d07a2c56d1ef8ba756, NAME => 'testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,470 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,471 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName a 2024-11-21T12:30:38,472 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,478 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/a/812dd7b250c94d55bfcef29ee753c6e0 2024-11-21T12:30:38,478 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,478 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,480 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName b 2024-11-21T12:30:38,480 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,480 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,481 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,482 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fdb275382ff63d07a2c56d1ef8ba756 columnFamilyName c 2024-11-21T12:30:38,482 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,488 DEBUG [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/c/168b4c7f06b54c76a537f7b096f2e4b9 2024-11-21T12:30:38,488 INFO [StoreOpener-7fdb275382ff63d07a2c56d1ef8ba756-1 {}] regionserver.HStore(327): Store=7fdb275382ff63d07a2c56d1ef8ba756/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,488 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,489 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,491 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 2024-11-21T12:30:38,522 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:38,524 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 2024-11-21T12:30:38,524 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7fdb275382ff63d07a2c56d1ef8ba756 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-21T12:30:38,540 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/ace70b575117450dbd8cd130cb5beafb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732192238025/Put/seqid=0 2024-11-21T12:30:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741905_1083 (size=5958) 2024-11-21T12:30:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741905_1083 (size=5958) 2024-11-21T12:30:38,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741905_1083 (size=5958) 2024-11-21T12:30:38,553 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/ace70b575117450dbd8cd130cb5beafb 2024-11-21T12:30:38,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/.tmp/b/ace70b575117450dbd8cd130cb5beafb as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/ace70b575117450dbd8cd130cb5beafb 2024-11-21T12:30:38,568 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/b/ace70b575117450dbd8cd130cb5beafb, entries=10, sequenceid=32, filesize=5.8 K 2024-11-21T12:30:38,568 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7fdb275382ff63d07a2c56d1ef8ba756 in 44ms, sequenceid=32, compaction requested=false; wal=null 2024-11-21T12:30:38,569 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/0000000000000000032 2024-11-21T12:30:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,571 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:38,573 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7fdb275382ff63d07a2c56d1ef8ba756 2024-11-21T12:30:38,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/7fdb275382ff63d07a2c56d1ef8ba756/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-21T12:30:38,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7fdb275382ff63d07a2c56d1ef8ba756; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62357039, jitterRate=-0.07080771028995514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7fdb275382ff63d07a2c56d1ef8ba756: Writing region info on filesystem at 1732192238469Initializing all the Stores at 1732192238470 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238470Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238470Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238470Obtaining lock to block concurrent updates at 1732192238524 (+54 ms)Preparing flush snapshotting stores in 7fdb275382ff63d07a2c56d1ef8ba756 at 1732192238524Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732192238524Flushing stores of testReplayEditsWrittenViaHRegion,,1732192237858.7fdb275382ff63d07a2c56d1ef8ba756. at 1732192238524Flushing 7fdb275382ff63d07a2c56d1ef8ba756/b: creating writer at 1732192238524Flushing 7fdb275382ff63d07a2c56d1ef8ba756/b: appending metadata at 1732192238540 (+16 ms)Flushing 7fdb275382ff63d07a2c56d1ef8ba756/b: closing flushed file at 1732192238540Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a8950ec: reopening flushed file at 1732192238559 (+19 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7fdb275382ff63d07a2c56d1ef8ba756 in 44ms, sequenceid=32, compaction requested=false; wal=null at 1732192238568 (+9 ms)Cleaning up temporary data from old regions at 1732192238570 (+2 ms)Region opened successfully at 1732192238577 (+7 ms) 2024-11-21T12:30:38,603 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:40172 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:42476 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:49348 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:35040 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:59676 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:50388 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1046) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=6988 (was 7021) 2024-11-21T12:30:38,603 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-21T12:30:38,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=6988 2024-11-21T12:30:38,618 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-21T12:30:38,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:38,637 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:38,638 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:38,641 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57195401, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-57195401, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:38,656 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57195401/hregion-57195401.1732192238641, exclude list is [], retry=0 2024-11-21T12:30:38,659 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:38,659 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:38,659 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:38,691 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57195401/hregion-57195401.1732192238641 2024-11-21T12:30:38,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:38,692 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 56cd064587ed1c6c07afeb0b8fad847b, NAME => 'testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:38,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741907_1085 (size=68) 2024-11-21T12:30:38,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741907_1085 (size=68) 2024-11-21T12:30:38,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741907_1085 (size=68) 2024-11-21T12:30:38,704 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:38,706 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,707 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName a 2024-11-21T12:30:38,708 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,708 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,708 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,710 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName b 2024-11-21T12:30:38,710 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,711 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,711 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,712 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName c 2024-11-21T12:30:38,712 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,713 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,713 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,715 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:38,717 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,720 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:38,720 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56cd064587ed1c6c07afeb0b8fad847b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59491754, jitterRate=-0.11350378394126892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56cd064587ed1c6c07afeb0b8fad847b: Writing region info on filesystem at 1732192238704Initializing all the Stores at 1732192238705 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238705Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238705Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238705Cleaning up temporary data from old regions at 1732192238715 (+10 ms)Region opened successfully at 1732192238721 (+6 ms) 2024-11-21T12:30:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56cd064587ed1c6c07afeb0b8fad847b, disabling compactions & flushes 2024-11-21T12:30:38,721 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. after waiting 0 ms 2024-11-21T12:30:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:38,722 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:38,722 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56cd064587ed1c6c07afeb0b8fad847b: Waiting for close lock at 1732192238721Disabling compacts and flushes for region at 1732192238721Disabling writes for close at 1732192238721Writing region close event to WAL at 1732192238722 (+1 ms)Closed at 1732192238722 2024-11-21T12:30:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741906_1084 (size=175) 2024-11-21T12:30:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741906_1084 (size=175) 2024-11-21T12:30:38,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741906_1084 (size=175) 2024-11-21T12:30:38,731 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:38,731 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57195401:(num 1732192238641) 2024-11-21T12:30:38,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:38,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:38,747 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, exclude list is [], retry=0 2024-11-21T12:30:38,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:38,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:38,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:38,781 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 2024-11-21T12:30:38,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:38,840 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 56cd064587ed1c6c07afeb0b8fad847b, NAME => 'testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:38,843 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,843 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:38,843 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,843 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,846 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,847 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName a 2024-11-21T12:30:38,847 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,848 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,848 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,849 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName b 2024-11-21T12:30:38,849 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,849 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,850 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,850 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName c 2024-11-21T12:30:38,851 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:38,851 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:38,851 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,852 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,855 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,855 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,855 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:38,857 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,858 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56cd064587ed1c6c07afeb0b8fad847b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61999109, jitterRate=-0.07614128291606903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:38,858 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:38,859 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56cd064587ed1c6c07afeb0b8fad847b: Running coprocessor pre-open hook at 1732192238843Writing region info on filesystem at 1732192238843Initializing all the Stores at 1732192238845 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238845Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238845Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192238845Cleaning up temporary data from old regions at 1732192238855 (+10 ms)Running coprocessor post-open hooks at 1732192238858 (+3 ms)Region opened successfully at 1732192238859 (+1 ms) 2024-11-21T12:30:38,872 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56cd064587ed1c6c07afeb0b8fad847b 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-21T12:30:38,873 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:39,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T12:30:39,874 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:40,874 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:41,875 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:42,876 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:43,876 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:44,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T12:30:44,747 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T12:30:44,877 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:45,878 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:46,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-21T12:30:46,207 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:46,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T12:30:46,208 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:46,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-21T12:30:46,209 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:46,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-21T12:30:46,209 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:46,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T12:30:46,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T12:30:46,878 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:47,879 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 56cd064587ed1c6c07afeb0b8fad847b/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:47,880 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 56cd064587ed1c6c07afeb0b8fad847b: 2024-11-21T12:30:47,880 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 56cd064587ed1c6c07afeb0b8fad847b: 2024-11-21T12:30:47,893 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-21T12:30:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56cd064587ed1c6c07afeb0b8fad847b, disabling compactions & flushes 2024-11-21T12:30:47,893 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. after waiting 0 ms 2024-11-21T12:30:47,893 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,894 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,894 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. 2024-11-21T12:30:47,894 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56cd064587ed1c6c07afeb0b8fad847b: Waiting for close lock at 1732192247893Running coprocessor pre-close hooks at 1732192247893Disabling compacts and flushes for region at 1732192247893Disabling writes for close at 1732192247893Writing region close event to WAL at 1732192247894 (+1 ms)Running coprocessor post-close hooks at 1732192247894Closed at 1732192247894 2024-11-21T12:30:47,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741908_1086 (size=4236) 2024-11-21T12:30:47,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741908_1086 (size=4236) 2024-11-21T12:30:47,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741908_1086 (size=4236) 2024-11-21T12:30:47,914 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, size=4.1 K (4236bytes) 2024-11-21T12:30:47,914 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 2024-11-21T12:30:47,915 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 after 1ms 2024-11-21T12:30:47,944 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:47,944 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 took 30ms 2024-11-21T12:30:47,946 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 so closing down 2024-11-21T12:30:47,947 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:47,948 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732192238733.temp 2024-11-21T12:30:47,980 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp 2024-11-21T12:30:47,980 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741909_1087 (size=3245) 2024-11-21T12:30:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741909_1087 (size=3245) 2024-11-21T12:30:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741909_1087 (size=3245) 2024-11-21T12:30:47,988 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:47,989 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 2024-11-21T12:30:47,989 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-21T12:30:47,990 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, journal: Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, size=4.1 K (4236bytes) at 1732192247914Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 so closing down at 1732192247947 (+33 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp at 1732192247980 (+33 ms)3 split writer threads finished at 1732192247980Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732192247988 (+8 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000004-wal.1732192238733.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 at 1732192247989 (+1 ms)Processed 23 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732192247989 2024-11-21T12:30:47,991 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192238733 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192238733 2024-11-21T12:30:47,992 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 2024-11-21T12:30:47,992 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:47,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:48,008 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192247995, exclude list is [], retry=0 2024-11-21T12:30:48,011 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:48,011 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:48,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:48,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192247995 2024-11-21T12:30:48,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 56cd064587ed1c6c07afeb0b8fad847b, NAME => 'testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:48,054 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:48,055 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,055 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,057 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,058 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName a 2024-11-21T12:30:48,058 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,058 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,058 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,059 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName b 2024-11-21T12:30:48,059 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,060 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,060 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,061 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56cd064587ed1c6c07afeb0b8fad847b columnFamilyName c 2024-11-21T12:30:48,061 DEBUG [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,061 INFO [StoreOpener-56cd064587ed1c6c07afeb0b8fad847b-1 {}] regionserver.HStore(327): Store=56cd064587ed1c6c07afeb0b8fad847b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,061 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,064 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 2024-11-21T12:30:48,093 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:48,121 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 2024-11-21T12:30:48,121 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56cd064587ed1c6c07afeb0b8fad847b 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-21T12:30:48,140 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/a/2cca02b14c3b4202a570f1c4f050603c is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732192247885/Put/seqid=0 2024-11-21T12:30:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741911_1089 (size=5523) 2024-11-21T12:30:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741911_1089 (size=5523) 2024-11-21T12:30:48,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741911_1089 (size=5523) 2024-11-21T12:30:48,149 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/a/2cca02b14c3b4202a570f1c4f050603c 2024-11-21T12:30:48,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/b/2ea2af5744b54334abf562ca37da1d72 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732192247881/Put/seqid=0 2024-11-21T12:30:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741912_1090 (size=5524) 2024-11-21T12:30:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741912_1090 (size=5524) 2024-11-21T12:30:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741912_1090 (size=5524) 2024-11-21T12:30:48,178 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/b/2ea2af5744b54334abf562ca37da1d72 2024-11-21T12:30:48,199 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/c/3add9371592540a495292b69791c895e is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732192247882/Put/seqid=0 2024-11-21T12:30:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741913_1091 (size=5457) 2024-11-21T12:30:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741913_1091 (size=5457) 2024-11-21T12:30:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741913_1091 (size=5457) 2024-11-21T12:30:48,208 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/c/3add9371592540a495292b69791c895e 2024-11-21T12:30:48,214 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/a/2cca02b14c3b4202a570f1c4f050603c as hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/a/2cca02b14c3b4202a570f1c4f050603c 2024-11-21T12:30:48,221 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/a/2cca02b14c3b4202a570f1c4f050603c, entries=7, sequenceid=26, filesize=5.4 K 2024-11-21T12:30:48,222 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/b/2ea2af5744b54334abf562ca37da1d72 as hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/b/2ea2af5744b54334abf562ca37da1d72 2024-11-21T12:30:48,229 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/b/2ea2af5744b54334abf562ca37da1d72, entries=7, sequenceid=26, filesize=5.4 K 2024-11-21T12:30:48,230 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/.tmp/c/3add9371592540a495292b69791c895e as hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/c/3add9371592540a495292b69791c895e 2024-11-21T12:30:48,235 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/c/3add9371592540a495292b69791c895e, entries=6, sequenceid=26, filesize=5.3 K 2024-11-21T12:30:48,236 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 56cd064587ed1c6c07afeb0b8fad847b in 115ms, sequenceid=26, compaction requested=false; wal=null 2024-11-21T12:30:48,237 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/0000000000000000026 2024-11-21T12:30:48,238 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,238 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,238 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:48,240 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,242 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsAfterAbortingFlush/56cd064587ed1c6c07afeb0b8fad847b/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-21T12:30:48,243 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56cd064587ed1c6c07afeb0b8fad847b; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63710858, jitterRate=-0.0506342351436615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:48,243 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56cd064587ed1c6c07afeb0b8fad847b 2024-11-21T12:30:48,244 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56cd064587ed1c6c07afeb0b8fad847b: Running coprocessor pre-open hook at 1732192248055Writing region info on filesystem at 1732192248055Initializing all the Stores at 1732192248056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248056Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248056Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248056Obtaining lock to block concurrent updates at 1732192248121 (+65 ms)Preparing flush snapshotting stores in 56cd064587ed1c6c07afeb0b8fad847b at 1732192248121Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732192248122 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732192238636.56cd064587ed1c6c07afeb0b8fad847b. at 1732192248122Flushing 56cd064587ed1c6c07afeb0b8fad847b/a: creating writer at 1732192248122Flushing 56cd064587ed1c6c07afeb0b8fad847b/a: appending metadata at 1732192248139 (+17 ms)Flushing 56cd064587ed1c6c07afeb0b8fad847b/a: closing flushed file at 1732192248139Flushing 56cd064587ed1c6c07afeb0b8fad847b/b: creating writer at 1732192248154 (+15 ms)Flushing 56cd064587ed1c6c07afeb0b8fad847b/b: appending metadata at 1732192248169 (+15 ms)Flushing 56cd064587ed1c6c07afeb0b8fad847b/b: closing flushed file at 1732192248169Flushing 56cd064587ed1c6c07afeb0b8fad847b/c: creating writer at 1732192248183 (+14 ms)Flushing 56cd064587ed1c6c07afeb0b8fad847b/c: appending metadata at 1732192248199 (+16 ms)Flushing 56cd064587ed1c6c07afeb0b8fad847b/c: closing flushed file at 1732192248199Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6edb24: reopening flushed file at 1732192248213 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8ac509e: reopening flushed file at 1732192248221 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19da3bcc: reopening flushed file at 1732192248229 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 56cd064587ed1c6c07afeb0b8fad847b in 115ms, sequenceid=26, compaction requested=false; wal=null at 1732192248236 (+7 ms)Cleaning up temporary data from old regions at 1732192248238 (+2 ms)Running coprocessor post-open hooks at 1732192248243 (+5 ms)Region opened successfully at 1732192248244 (+1 ms) 2024-11-21T12:30:48,268 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 415) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43728 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43700 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:45790 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51726 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:45804 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51744 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=6977 (was 6988) 2024-11-21T12:30:48,268 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-21T12:30:48,282 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=6976 2024-11-21T12:30:48,282 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-21T12:30:48,297 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:48,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:48,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:48,302 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55461729, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-55461729, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:48,316 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55461729/hregion-55461729.1732192248302, exclude list is [], retry=0 2024-11-21T12:30:48,318 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:48,319 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:48,319 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:48,361 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55461729/hregion-55461729.1732192248302 2024-11-21T12:30:48,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:48,361 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e11b2b901be295841190817fa3172ac3, NAME => 'testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:48,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741915_1093 (size=61) 2024-11-21T12:30:48,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741915_1093 (size=61) 2024-11-21T12:30:48,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741915_1093 (size=61) 2024-11-21T12:30:48,373 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:48,375 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,376 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e11b2b901be295841190817fa3172ac3 columnFamilyName a 2024-11-21T12:30:48,376 DEBUG [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,377 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(327): Store=e11b2b901be295841190817fa3172ac3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,377 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,377 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,380 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,382 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:48,382 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e11b2b901be295841190817fa3172ac3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65158730, jitterRate=-0.029059261083602905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:48,383 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e11b2b901be295841190817fa3172ac3: Writing region info on filesystem at 1732192248373Initializing all the Stores at 1732192248374 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248374Cleaning up temporary data from old regions at 1732192248378 (+4 ms)Region opened successfully at 1732192248383 (+5 ms) 2024-11-21T12:30:48,383 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e11b2b901be295841190817fa3172ac3, disabling compactions & flushes 2024-11-21T12:30:48,383 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,383 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,383 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. after waiting 0 ms 2024-11-21T12:30:48,383 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,384 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,384 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e11b2b901be295841190817fa3172ac3: Waiting for close lock at 1732192248383Disabling compacts and flushes for region at 1732192248383Disabling writes for close at 1732192248383Writing region close event to WAL at 1732192248384 (+1 ms)Closed at 1732192248384 2024-11-21T12:30:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741914_1092 (size=175) 2024-11-21T12:30:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741914_1092 (size=175) 2024-11-21T12:30:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741914_1092 (size=175) 2024-11-21T12:30:48,389 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:48,389 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55461729:(num 1732192248302) 2024-11-21T12:30:48,389 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:48,391 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:48,404 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, exclude list is [], retry=0 2024-11-21T12:30:48,407 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:48,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:48,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:48,444 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 2024-11-21T12:30:48,445 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:48,445 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e11b2b901be295841190817fa3172ac3, NAME => 'testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:48,445 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:48,445 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,445 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,447 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,447 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e11b2b901be295841190817fa3172ac3 columnFamilyName a 2024-11-21T12:30:48,448 DEBUG [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,448 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(327): Store=e11b2b901be295841190817fa3172ac3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,448 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,450 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,450 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,450 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,452 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,452 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e11b2b901be295841190817fa3172ac3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68330185, jitterRate=0.018199101090431213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:48,453 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e11b2b901be295841190817fa3172ac3: Writing region info on filesystem at 1732192248445Initializing all the Stores at 1732192248446 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248446Cleaning up temporary data from old regions at 1732192248450 (+4 ms)Region opened successfully at 1732192248453 (+3 ms) 2024-11-21T12:30:48,462 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e11b2b901be295841190817fa3172ac3, disabling compactions & flushes 2024-11-21T12:30:48,462 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,462 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,462 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. after waiting 0 ms 2024-11-21T12:30:48,462 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,463 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,463 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. 2024-11-21T12:30:48,463 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e11b2b901be295841190817fa3172ac3: Waiting for close lock at 1732192248462Disabling compacts and flushes for region at 1732192248462Disabling writes for close at 1732192248462Writing region close event to WAL at 1732192248463 (+1 ms)Closed at 1732192248463 2024-11-21T12:30:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741916_1094 (size=1795) 2024-11-21T12:30:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741916_1094 (size=1795) 2024-11-21T12:30:48,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741916_1094 (size=1795) 2024-11-21T12:30:48,482 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, size=1.8 K (1795bytes) 2024-11-21T12:30:48,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 2024-11-21T12:30:48,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 after 0ms 2024-11-21T12:30:48,511 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:48,511 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 took 29ms 2024-11-21T12:30:48,513 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 so closing down 2024-11-21T12:30:48,513 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:48,515 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732192248391.temp 2024-11-21T12:30:48,542 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp 2024-11-21T12:30:48,543 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741917_1095 (size=1795) 2024-11-21T12:30:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741917_1095 (size=1795) 2024-11-21T12:30:48,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741917_1095 (size=1795) 2024-11-21T12:30:48,551 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:48,552 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp to hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 2024-11-21T12:30:48,553 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-21T12:30:48,553 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, journal: Splitting hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, size=1.8 K (1795bytes) at 1732192248482Finishing writing output for hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 so closing down at 1732192248513 (+31 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp at 1732192248542 (+29 ms)3 split writer threads finished at 1732192248543 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732192248551 (+8 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000003-wal.1732192248391.temp to hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 at 1732192248552 (+1 ms)Processed 10 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732192248553 (+1 ms) 2024-11-21T12:30:48,554 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248391 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192248391 2024-11-21T12:30:48,555 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 2024-11-21T12:30:48,586 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:48,910 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:48,913 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:48,927 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248913, exclude list is [], retry=0 2024-11-21T12:30:48,929 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:48,930 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:48,930 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:48,959 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248913 2024-11-21T12:30:48,959 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:48,959 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e11b2b901be295841190817fa3172ac3, NAME => 'testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:48,960 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:48,960 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,960 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,963 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,964 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e11b2b901be295841190817fa3172ac3 columnFamilyName a 2024-11-21T12:30:48,965 DEBUG [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:48,965 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(327): Store=e11b2b901be295841190817fa3172ac3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:48,966 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:48,969 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 2024-11-21T12:30:48,997 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:48,999 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 2024-11-21T12:30:48,999 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e11b2b901be295841190817fa3172ac3 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-21T12:30:49,018 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/.tmp/a/ff797b4112d445818ff261683a9fd201 is 79, key is testDatalossWhenInputError/a:x0/1732192248453/Put/seqid=0 2024-11-21T12:30:49,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741919_1097 (size=5808) 2024-11-21T12:30:49,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741919_1097 (size=5808) 2024-11-21T12:30:49,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741919_1097 (size=5808) 2024-11-21T12:30:49,028 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/.tmp/a/ff797b4112d445818ff261683a9fd201 2024-11-21T12:30:49,037 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/.tmp/a/ff797b4112d445818ff261683a9fd201 as hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/a/ff797b4112d445818ff261683a9fd201 2024-11-21T12:30:49,046 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/a/ff797b4112d445818ff261683a9fd201, entries=10, sequenceid=12, filesize=5.7 K 2024-11-21T12:30:49,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e11b2b901be295841190817fa3172ac3 in 47ms, sequenceid=12, compaction requested=false; wal=null 2024-11-21T12:30:49,047 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/0000000000000000012 2024-11-21T12:30:49,048 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,048 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,051 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,054 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-21T12:30:49,055 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e11b2b901be295841190817fa3172ac3; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65272211, jitterRate=-0.02736826241016388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:49,056 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e11b2b901be295841190817fa3172ac3: Writing region info on filesystem at 1732192248960Initializing all the Stores at 1732192248962 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192248962Obtaining lock to block concurrent updates at 1732192248999 (+37 ms)Preparing flush snapshotting stores in e11b2b901be295841190817fa3172ac3 at 1732192248999Finished memstore snapshotting testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732192248999Flushing stores of testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3. at 1732192248999Flushing e11b2b901be295841190817fa3172ac3/a: creating writer at 1732192248999Flushing e11b2b901be295841190817fa3172ac3/a: appending metadata at 1732192249017 (+18 ms)Flushing e11b2b901be295841190817fa3172ac3/a: closing flushed file at 1732192249017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4210620c: reopening flushed file at 1732192249035 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e11b2b901be295841190817fa3172ac3 in 47ms, sequenceid=12, compaction requested=false; wal=null at 1732192249046 (+11 ms)Cleaning up temporary data from old regions at 1732192249048 (+2 ms)Region opened successfully at 1732192249056 (+8 ms) 2024-11-21T12:30:49,060 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e11b2b901be295841190817fa3172ac3, NAME => 'testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:49,060 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732192248297.e11b2b901be295841190817fa3172ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:49,060 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,060 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,062 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,063 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e11b2b901be295841190817fa3172ac3 columnFamilyName a 2024-11-21T12:30:49,063 DEBUG [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,070 DEBUG [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/a/ff797b4112d445818ff261683a9fd201 2024-11-21T12:30:49,070 INFO [StoreOpener-e11b2b901be295841190817fa3172ac3-1 {}] regionserver.HStore(327): Store=e11b2b901be295841190817fa3172ac3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,070 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,071 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,075 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e11b2b901be295841190817fa3172ac3 2024-11-21T12:30:49,078 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testDatalossWhenInputError/e11b2b901be295841190817fa3172ac3/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-21T12:30:49,080 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e11b2b901be295841190817fa3172ac3; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61285889, jitterRate=-0.08676908910274506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T12:30:49,081 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e11b2b901be295841190817fa3172ac3: Writing region info on filesystem at 1732192249060Initializing all the Stores at 1732192249061 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249062 (+1 ms)Cleaning up temporary data from old regions at 1732192249073 (+11 ms)Region opened successfully at 1732192249081 (+8 ms) 2024-11-21T12:30:49,103 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=433 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43728 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43772 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51790 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51726 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:45894 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1266 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=6967 (was 6976) 2024-11-21T12:30:49,104 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-21T12:30:49,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=433, OpenFileDescriptor=1266, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=6967 2024-11-21T12:30:49,120 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-21T12:30:49,142 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:49,144 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:49,145 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:49,148 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71914321, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-71914321, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:49,164 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71914321/hregion-71914321.1732192249148, exclude list is [], retry=0 2024-11-21T12:30:49,167 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:49,168 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:49,169 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:49,198 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71914321/hregion-71914321.1732192249148 2024-11-21T12:30:49,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:49,199 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 656f70e76000eb93df5eda2645e7af57, NAME => 'testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:49,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741921_1099 (size=63) 2024-11-21T12:30:49,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741921_1099 (size=63) 2024-11-21T12:30:49,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741921_1099 (size=63) 2024-11-21T12:30:49,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:49,210 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,212 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName a 2024-11-21T12:30:49,212 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,212 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,213 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,214 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName b 2024-11-21T12:30:49,214 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,214 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,215 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,216 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName c 2024-11-21T12:30:49,216 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,216 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,217 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,217 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,218 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,219 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,219 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,219 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:49,221 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,223 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:49,223 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 656f70e76000eb93df5eda2645e7af57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63597818, jitterRate=-0.05231866240501404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:49,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 656f70e76000eb93df5eda2645e7af57: Writing region info on filesystem at 1732192249209Initializing all the Stores at 1732192249210 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249210Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249210Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249210Cleaning up temporary data from old regions at 1732192249219 (+9 ms)Region opened successfully at 1732192249224 (+5 ms) 2024-11-21T12:30:49,224 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 656f70e76000eb93df5eda2645e7af57, disabling compactions & flushes 2024-11-21T12:30:49,224 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:49,224 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:49,224 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. after waiting 0 ms 2024-11-21T12:30:49,224 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:49,225 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:49,225 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 656f70e76000eb93df5eda2645e7af57: Waiting for close lock at 1732192249224Disabling compacts and flushes for region at 1732192249224Disabling writes for close at 1732192249224Writing region close event to WAL at 1732192249225 (+1 ms)Closed at 1732192249225 2024-11-21T12:30:49,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741920_1098 (size=175) 2024-11-21T12:30:49,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741920_1098 (size=175) 2024-11-21T12:30:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741920_1098 (size=175) 2024-11-21T12:30:49,230 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:49,230 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71914321:(num 1732192249148) 2024-11-21T12:30:49,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:49,232 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:49,248 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, exclude list is [], retry=0 2024-11-21T12:30:49,251 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:49,252 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:49,252 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:49,282 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 2024-11-21T12:30:49,283 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:49,283 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 656f70e76000eb93df5eda2645e7af57, NAME => 'testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:49,283 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:49,283 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,283 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,285 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,286 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName a 2024-11-21T12:30:49,286 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,286 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,286 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,287 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName b 2024-11-21T12:30:49,287 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,288 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,288 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,289 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName c 2024-11-21T12:30:49,289 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:49,289 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:49,289 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,290 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,292 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,292 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,293 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:49,294 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:49,295 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 656f70e76000eb93df5eda2645e7af57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70510384, jitterRate=0.05068659782409668}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:49,296 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 656f70e76000eb93df5eda2645e7af57: Writing region info on filesystem at 1732192249283Initializing all the Stores at 1732192249284 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249284Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249284Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192249285 (+1 ms)Cleaning up temporary data from old regions at 1732192249292 (+7 ms)Region opened successfully at 1732192249296 (+4 ms) 2024-11-21T12:30:49,300 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732192249300/Put/seqid=0 2024-11-21T12:30:49,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741923_1101 (size=4875) 2024-11-21T12:30:49,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741923_1101 (size=4875) 2024-11-21T12:30:49,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741923_1101 (size=4875) 2024-11-21T12:30:49,318 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732192249317/Put/seqid=0 2024-11-21T12:30:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741924_1102 (size=4875) 2024-11-21T12:30:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741924_1102 (size=4875) 2024-11-21T12:30:49,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741924_1102 (size=4875) 2024-11-21T12:30:49,329 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732192249329/Put/seqid=0 2024-11-21T12:30:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741925_1103 (size=4875) 2024-11-21T12:30:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741925_1103 (size=4875) 2024-11-21T12:30:49,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741925_1103 (size=4875) 2024-11-21T12:30:49,344 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,349 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-21T12:30:49,349 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-21T12:30:49,349 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,354 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-21T12:30:49,354 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-21T12:30:49,354 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,358 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-21T12:30:49,358 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-21T12:30:49,359 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 656f70e76000eb93df5eda2645e7af57 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-21T12:30:49,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp/a/32cd1c1377434897a0b503747d241b91 is 55, key is testCompactedBulkLoadedFiles/a:a/1732192249296/Put/seqid=0 2024-11-21T12:30:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741926_1104 (size=5107) 2024-11-21T12:30:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741926_1104 (size=5107) 2024-11-21T12:30:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741926_1104 (size=5107) 2024-11-21T12:30:49,388 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp/a/32cd1c1377434897a0b503747d241b91 2024-11-21T12:30:49,395 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp/a/32cd1c1377434897a0b503747d241b91 as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91 2024-11-21T12:30:49,401 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91, entries=1, sequenceid=4, filesize=5.0 K 2024-11-21T12:30:49,402 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 656f70e76000eb93df5eda2645e7af57 in 43ms, sequenceid=4, compaction requested=false 2024-11-21T12:30:49,402 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 656f70e76000eb93df5eda2645e7af57: 2024-11-21T12:30:49,404 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ 2024-11-21T12:30:49,405 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ 2024-11-21T12:30:49,406 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ 2024-11-21T12:30:49,407 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile0 into 656f70e76000eb93df5eda2645e7af57/a as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ - updating store file list. 2024-11-21T12:30:49,413 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc621ea630574723a5b214a62b4b67c5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:49,413 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ into 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,413 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile0 into 656f70e76000eb93df5eda2645e7af57/a (new location: hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_) 2024-11-21T12:30:49,414 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile1 into 656f70e76000eb93df5eda2645e7af57/a as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ - updating store file list. 2024-11-21T12:30:49,420 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:49,420 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ into 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,420 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile1 into 656f70e76000eb93df5eda2645e7af57/a (new location: hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_) 2024-11-21T12:30:49,421 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile2 into 656f70e76000eb93df5eda2645e7af57/a as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ - updating store file list. 2024-11-21T12:30:49,426 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 49898af76a254cf5a86797f20175e833_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:49,426 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ into 656f70e76000eb93df5eda2645e7af57/a 2024-11-21T12:30:49,426 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41319/hbase/testCompactedBulkLoadedFiles/hfile2 into 656f70e76000eb93df5eda2645e7af57/a (new location: hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_) 2024-11-21T12:30:49,434 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T12:30:49,434 DEBUG [Time-limited test {}] regionserver.HStore(1541): 656f70e76000eb93df5eda2645e7af57/a is initiating major compaction (all files) 2024-11-21T12:30:49,435 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 656f70e76000eb93df5eda2645e7af57/a in testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:49,435 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_] into tmpdir=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp, totalSize=19.3 K 2024-11-21T12:30:49,435 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 32cd1c1377434897a0b503747d241b91, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732192249296 2024-11-21T12:30:49,436 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dc621ea630574723a5b214a62b4b67c5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-21T12:30:49,436 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-21T12:30:49,437 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 49898af76a254cf5a86797f20175e833_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-21T12:30:49,452 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp/a/2073df19273347879ffa1e749c612dd2 is 55, key is testCompactedBulkLoadedFiles/a:a/1732192249296/Put/seqid=0 2024-11-21T12:30:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741927_1105 (size=6154) 2024-11-21T12:30:49,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741927_1105 (size=6154) 2024-11-21T12:30:49,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741927_1105 (size=6154) 2024-11-21T12:30:49,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/.tmp/a/2073df19273347879ffa1e749c612dd2 as hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/2073df19273347879ffa1e749c612dd2 2024-11-21T12:30:49,478 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 656f70e76000eb93df5eda2645e7af57/a of 656f70e76000eb93df5eda2645e7af57 into 2073df19273347879ffa1e749c612dd2(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T12:30:49,478 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 656f70e76000eb93df5eda2645e7af57: 2024-11-21T12:30:49,478 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-21T12:30:49,478 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-21T12:30:49,513 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, size=0 (0bytes) 2024-11-21T12:30:49,514 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 might be still open, length is 0 2024-11-21T12:30:49,514 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 2024-11-21T12:30:49,514 WARN [IPC Server handler 1 on default port 41319 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-21T12:30:49,515 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 after 1ms 2024-11-21T12:30:50,513 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51818 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37507:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51818 dst: /127.0.0.1:37507 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37507 remote=/127.0.0.1:51818]. Total timeout mills is 60000, 58965 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:50,514 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:45934 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34335:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45934 dst: /127.0.0.1:34335 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:50,514 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43798 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43798 dst: /127.0.0.1:39871 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:50,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741922_1106 (size=1794) 2024-11-21T12:30:50,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741922_1106 (size=1794) 2024-11-21T12:30:53,396 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T12:30:53,515 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 after 4001ms 2024-11-21T12:30:53,544 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:53,544 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 took 4031ms 2024-11-21T12:30:53,546 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232; continuing. 2024-11-21T12:30:53,546 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 so closing down 2024-11-21T12:30:53,546 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:53,547 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732192249232.temp 2024-11-21T12:30:53,574 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp 2024-11-21T12:30:53,574 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:53,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741928_1107 (size=800) 2024-11-21T12:30:53,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741928_1107 (size=800) 2024-11-21T12:30:53,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741928_1107 (size=800) 2024-11-21T12:30:53,581 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:53,583 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp to hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 2024-11-21T12:30:53,583 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, size=0, length=0, corrupted=false, cancelled=false 2024-11-21T12:30:53,583 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, journal: Splitting hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, size=0 (0bytes) at 1732192249514Finishing writing output for hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 so closing down at 1732192253546 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp at 1732192253574 (+28 ms)3 split writer threads finished at 1732192253574Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732192253581 (+7 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000003-wal.1732192249232.temp to hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 at 1732192253583 (+2 ms)Processed 5 edits across 1 Regions in 39 ms; skipped=3; WAL=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232, size=0, length=0, corrupted=false, cancelled=false at 1732192253583 2024-11-21T12:30:53,585 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192249232 2024-11-21T12:30:53,586 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 2024-11-21T12:30:53,586 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:53,588 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:53,608 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192253588, exclude list is [], retry=0 2024-11-21T12:30:53,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:53,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:53,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:53,639 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192253588 2024-11-21T12:30:53,640 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:53,640 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 656f70e76000eb93df5eda2645e7af57, NAME => 'testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:53,640 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:53,640 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,640 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,642 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,642 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName a 2024-11-21T12:30:53,643 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,649 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/2073df19273347879ffa1e749c612dd2 2024-11-21T12:30:53,653 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91 2024-11-21T12:30:53,656 DEBUG [StoreFileOpener-656f70e76000eb93df5eda2645e7af57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 49898af76a254cf5a86797f20175e833_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:53,656 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ 2024-11-21T12:30:53,659 DEBUG [StoreFileOpener-656f70e76000eb93df5eda2645e7af57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:53,659 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ 2024-11-21T12:30:53,662 DEBUG [StoreFileOpener-656f70e76000eb93df5eda2645e7af57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc621ea630574723a5b214a62b4b67c5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-21T12:30:53,662 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ 2024-11-21T12:30:53,662 WARN [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6dc42602 2024-11-21T12:30:53,662 WARN [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6dc42602 2024-11-21T12:30:53,663 WARN [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6dc42602 2024-11-21T12:30:53,663 WARN [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6dc42602 2024-11-21T12:30:53,663 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_] to archive 2024-11-21T12:30:53,663 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T12:30:53,665 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91 to hdfs://localhost:41319/hbase/archive/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/32cd1c1377434897a0b503747d241b91 2024-11-21T12:30:53,666 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ to hdfs://localhost:41319/hbase/archive/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/49898af76a254cf5a86797f20175e833_SeqId_4_ 2024-11-21T12:30:53,667 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ to hdfs://localhost:41319/hbase/archive/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_ 2024-11-21T12:30:53,668 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ to hdfs://localhost:41319/hbase/archive/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/a/dc621ea630574723a5b214a62b4b67c5_SeqId_4_ 2024-11-21T12:30:53,668 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,668 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,669 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName b 2024-11-21T12:30:53,669 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,669 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,670 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,670 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 656f70e76000eb93df5eda2645e7af57 columnFamilyName c 2024-11-21T12:30:53,670 DEBUG [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,671 INFO [StoreOpener-656f70e76000eb93df5eda2645e7af57-1 {}] regionserver.HStore(327): Store=656f70e76000eb93df5eda2645e7af57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,671 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,672 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,673 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 2024-11-21T12:30:53,702 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:53,704 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 656f70e76000eb93df5eda2645e7af57 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "656f70e76000eb93df5eda2645e7af57" family_name: "a" compaction_input: "32cd1c1377434897a0b503747d241b91" compaction_input: "dc621ea630574723a5b214a62b4b67c5_SeqId_4_" compaction_input: "7d774e5b6d0c4f4aa2b16f3a1b536cda_SeqId_4_" compaction_input: "49898af76a254cf5a86797f20175e833_SeqId_4_" compaction_output: "2073df19273347879ffa1e749c612dd2" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-21T12:30:53,704 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-21T12:30:53,704 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 2024-11-21T12:30:53,705 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/0000000000000000008 2024-11-21T12:30:53,706 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,706 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,707 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:53,708 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 656f70e76000eb93df5eda2645e7af57 2024-11-21T12:30:53,710 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testCompactedBulkLoadedFiles/656f70e76000eb93df5eda2645e7af57/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-21T12:30:53,711 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 656f70e76000eb93df5eda2645e7af57; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61119962, jitterRate=-0.08924159407615662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:53,711 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 656f70e76000eb93df5eda2645e7af57: Writing region info on filesystem at 1732192253640Initializing all the Stores at 1732192253641 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253641Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253641Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253641Cleaning up temporary data from old regions at 1732192253706 (+65 ms)Region opened successfully at 1732192253711 (+5 ms) 2024-11-21T12:30:53,714 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 656f70e76000eb93df5eda2645e7af57, disabling compactions & flushes 2024-11-21T12:30:53,714 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:53,714 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:53,714 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. after waiting 0 ms 2024-11-21T12:30:53,714 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:53,715 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732192249143.656f70e76000eb93df5eda2645e7af57. 2024-11-21T12:30:53,715 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 656f70e76000eb93df5eda2645e7af57: Waiting for close lock at 1732192253714Disabling compacts and flushes for region at 1732192253714Disabling writes for close at 1732192253714Writing region close event to WAL at 1732192253715 (+1 ms)Closed at 1732192253715 2024-11-21T12:30:53,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741929_1108 (size=175) 2024-11-21T12:30:53,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741929_1108 (size=175) 2024-11-21T12:30:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741929_1108 (size=175) 2024-11-21T12:30:53,719 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:53,720 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732192253588) 2024-11-21T12:30:53,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=442 (was 433) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41319 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2063961131_22 at /127.0.0.1:46034 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2063961131_22 at /127.0.0.1:51878 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41319 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2063961131_22 at /127.0.0.1:43866 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1266) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=6949 (was 6967) 2024-11-21T12:30:53,736 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-21T12:30:53,751 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=6946 2024-11-21T12:30:53,751 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-21T12:30:53,766 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:53,768 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T12:30:53,768 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T12:30:53,771 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46356589, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/hregion-46356589, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:53,784 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46356589/hregion-46356589.1732192253771, exclude list is [], retry=0 2024-11-21T12:30:53,787 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:53,787 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:53,788 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:53,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46356589/hregion-46356589.1732192253771 2024-11-21T12:30:53,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699)] 2024-11-21T12:30:53,817 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cbb3fad630b08d16eeb6e185c159b392, NAME => 'testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41319/hbase 2024-11-21T12:30:53,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741931_1110 (size=67) 2024-11-21T12:30:53,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741931_1110 (size=67) 2024-11-21T12:30:53,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741931_1110 (size=67) 2024-11-21T12:30:53,830 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:53,831 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,833 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName a 2024-11-21T12:30:53,833 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,833 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,833 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,835 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName b 2024-11-21T12:30:53,835 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,835 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,835 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,836 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName c 2024-11-21T12:30:53,836 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,837 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,839 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,839 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,839 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:53,840 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,842 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T12:30:53,842 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cbb3fad630b08d16eeb6e185c159b392; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68157741, jitterRate=0.015629485249519348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cbb3fad630b08d16eeb6e185c159b392: Writing region info on filesystem at 1732192253830Initializing all the Stores at 1732192253831 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253831Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253831Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253831Cleaning up temporary data from old regions at 1732192253839 (+8 ms)Region opened successfully at 1732192253843 (+4 ms) 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cbb3fad630b08d16eeb6e185c159b392, disabling compactions & flushes 2024-11-21T12:30:53,843 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. after waiting 0 ms 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,843 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,843 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cbb3fad630b08d16eeb6e185c159b392: Waiting for close lock at 1732192253843Disabling compacts and flushes for region at 1732192253843Disabling writes for close at 1732192253843Writing region close event to WAL at 1732192253843Closed at 1732192253843 2024-11-21T12:30:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741930_1109 (size=175) 2024-11-21T12:30:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741930_1109 (size=175) 2024-11-21T12:30:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741930_1109 (size=175) 2024-11-21T12:30:53,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:53,848 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46356589:(num 1732192253771) 2024-11-21T12:30:53,848 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:53,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:53,866 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, exclude list is [], retry=0 2024-11-21T12:30:53,869 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:53,869 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:53,869 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:53,897 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 2024-11-21T12:30:53,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33707:33707),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:44529:44529)] 2024-11-21T12:30:53,898 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cbb3fad630b08d16eeb6e185c159b392, NAME => 'testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:53,898 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:53,898 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,898 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,900 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,901 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName a 2024-11-21T12:30:53,901 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,902 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,902 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,903 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName b 2024-11-21T12:30:53,903 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,903 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,903 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,904 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName c 2024-11-21T12:30:53,904 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:53,904 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:53,904 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,907 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,907 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,908 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:53,909 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:53,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cbb3fad630b08d16eeb6e185c159b392; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69075850, jitterRate=0.029310375452041626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:53,910 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cbb3fad630b08d16eeb6e185c159b392: Writing region info on filesystem at 1732192253898Initializing all the Stores at 1732192253899 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253899Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253900 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192253900Cleaning up temporary data from old regions at 1732192253907 (+7 ms)Region opened successfully at 1732192253910 (+3 ms) 2024-11-21T12:30:53,917 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cbb3fad630b08d16eeb6e185c159b392 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-21T12:30:53,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/16e16ba4c759459f906ac35c0a1b1915 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732192253910/Put/seqid=0 2024-11-21T12:30:53,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741933_1112 (size=5958) 2024-11-21T12:30:53,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741933_1112 (size=5958) 2024-11-21T12:30:53,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741933_1112 (size=5958) 2024-11-21T12:30:53,941 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/16e16ba4c759459f906ac35c0a1b1915 2024-11-21T12:30:53,947 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/16e16ba4c759459f906ac35c0a1b1915 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/16e16ba4c759459f906ac35c0a1b1915 2024-11-21T12:30:53,951 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/16e16ba4c759459f906ac35c0a1b1915, entries=10, sequenceid=13, filesize=5.8 K 2024-11-21T12:30:53,953 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for cbb3fad630b08d16eeb6e185c159b392 in 35ms, sequenceid=13, compaction requested=false 2024-11-21T12:30:53,953 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cbb3fad630b08d16eeb6e185c159b392: 2024-11-21T12:30:53,967 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cbb3fad630b08d16eeb6e185c159b392, disabling compactions & flushes 2024-11-21T12:30:53,968 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,968 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,968 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. after waiting 0 ms 2024-11-21T12:30:53,968 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,968 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,969 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:53,969 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cbb3fad630b08d16eeb6e185c159b392: Waiting for close lock at 1732192253967Disabling compacts and flushes for region at 1732192253967Disabling writes for close at 1732192253968 (+1 ms)Writing region close event to WAL at 1732192253968Closed at 1732192253969 (+1 ms) 2024-11-21T12:30:53,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741932_1111 (size=6243) 2024-11-21T12:30:53,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741932_1111 (size=6243) 2024-11-21T12:30:53,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741932_1111 (size=6243) 2024-11-21T12:30:53,987 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, size=6.1 K (6243bytes) 2024-11-21T12:30:53,987 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 2024-11-21T12:30:53,988 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 after 1ms 2024-11-21T12:30:54,015 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:54,015 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 took 28ms 2024-11-21T12:30:54,017 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 so closing down 2024-11-21T12:30:54,017 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:54,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732192253851.temp 2024-11-21T12:30:54,046 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp 2024-11-21T12:30:54,046 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:54,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741934_1113 (size=5575) 2024-11-21T12:30:54,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741934_1113 (size=5575) 2024-11-21T12:30:54,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741934_1113 (size=5575) 2024-11-21T12:30:54,055 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:54,057 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 2024-11-21T12:30:54,057 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-21T12:30:54,057 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, journal: Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, size=6.1 K (6243bytes) at 1732192253987Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 so closing down at 1732192254017 (+30 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp at 1732192254046 (+29 ms)3 split writer threads finished at 1732192254046Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732192254055 (+9 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000003-wal.1732192253851.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 at 1732192254057 (+2 ms)Processed 32 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732192254057 2024-11-21T12:30:54,059 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192253851 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192253851 2024-11-21T12:30:54,060 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 2024-11-21T12:30:54,060 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:54,062 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:54,075 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, exclude list is [], retry=0 2024-11-21T12:30:54,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:54,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:54,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:54,105 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 2024-11-21T12:30:54,105 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:54,105 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cbb3fad630b08d16eeb6e185c159b392, NAME => 'testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.', STARTKEY => '', ENDKEY => ''} 2024-11-21T12:30:54,105 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:54,105 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,105 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,107 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,107 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName a 2024-11-21T12:30:54,108 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:54,113 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/16e16ba4c759459f906ac35c0a1b1915 2024-11-21T12:30:54,113 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:54,113 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,114 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName b 2024-11-21T12:30:54,114 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:54,114 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:54,115 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,115 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName c 2024-11-21T12:30:54,115 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:54,116 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:54,116 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,117 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,118 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,119 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 2024-11-21T12:30:54,148 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:54,149 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 2024-11-21T12:30:54,149 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cbb3fad630b08d16eeb6e185c159b392 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-21T12:30:54,164 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/2e630fe646f14bdabc1faad85d2a8e43 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732192253953/Put/seqid=0 2024-11-21T12:30:54,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741936_1115 (size=5958) 2024-11-21T12:30:54,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741936_1115 (size=5958) 2024-11-21T12:30:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741936_1115 (size=5958) 2024-11-21T12:30:54,172 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/2e630fe646f14bdabc1faad85d2a8e43 2024-11-21T12:30:54,191 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/409a03d8a2984db180350cea01d9b52d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732192253959/Put/seqid=0 2024-11-21T12:30:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741937_1116 (size=5958) 2024-11-21T12:30:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741937_1116 (size=5958) 2024-11-21T12:30:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741937_1116 (size=5958) 2024-11-21T12:30:54,199 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/409a03d8a2984db180350cea01d9b52d 2024-11-21T12:30:54,205 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/2e630fe646f14bdabc1faad85d2a8e43 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/b/2e630fe646f14bdabc1faad85d2a8e43 2024-11-21T12:30:54,210 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/b/2e630fe646f14bdabc1faad85d2a8e43, entries=10, sequenceid=35, filesize=5.8 K 2024-11-21T12:30:54,211 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/409a03d8a2984db180350cea01d9b52d as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/c/409a03d8a2984db180350cea01d9b52d 2024-11-21T12:30:54,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/c/409a03d8a2984db180350cea01d9b52d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-21T12:30:54,216 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for cbb3fad630b08d16eeb6e185c159b392 in 67ms, sequenceid=35, compaction requested=false; wal=null 2024-11-21T12:30:54,216 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000035 2024-11-21T12:30:54,217 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,217 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,218 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:54,219 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:54,221 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-21T12:30:54,222 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cbb3fad630b08d16eeb6e185c159b392; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74312528, jitterRate=0.10734295845031738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:54,222 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cbb3fad630b08d16eeb6e185c159b392: Writing region info on filesystem at 1732192254106Initializing all the Stores at 1732192254106Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192254106Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192254107 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192254107Obtaining lock to block concurrent updates at 1732192254149 (+42 ms)Preparing flush snapshotting stores in cbb3fad630b08d16eeb6e185c159b392 at 1732192254149Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732192254149Flushing stores of testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. at 1732192254149Flushing cbb3fad630b08d16eeb6e185c159b392/b: creating writer at 1732192254149Flushing cbb3fad630b08d16eeb6e185c159b392/b: appending metadata at 1732192254163 (+14 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/b: closing flushed file at 1732192254163Flushing cbb3fad630b08d16eeb6e185c159b392/c: creating writer at 1732192254177 (+14 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/c: appending metadata at 1732192254191 (+14 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/c: closing flushed file at 1732192254191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c15a801: reopening flushed file at 1732192254204 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797a22b6: reopening flushed file at 1732192254210 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for cbb3fad630b08d16eeb6e185c159b392 in 67ms, sequenceid=35, compaction requested=false; wal=null at 1732192254216 (+6 ms)Cleaning up temporary data from old regions at 1732192254217 (+1 ms)Region opened successfully at 1732192254222 (+5 ms) 2024-11-21T12:30:54,288 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, size=0 (0bytes) 2024-11-21T12:30:54,288 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 might be still open, length is 0 2024-11-21T12:30:54,288 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 2024-11-21T12:30:54,289 WARN [IPC Server handler 4 on default port 41319 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-21T12:30:54,289 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 after 1ms 2024-11-21T12:30:56,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-21T12:30:56,207 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:56,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-21T12:30:56,208 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-21T12:30:56,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:43930 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39871:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43930 dst: /127.0.0.1:39871 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39871 remote=/127.0.0.1:43930]. Total timeout mills is 60000, 57738 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:56,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:51960 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37507:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51960 dst: /127.0.0.1:37507 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:56,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1519374329_22 at /127.0.0.1:46094 [Receiving block BP-1168079295-172.17.0.2-1732192212418:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34335:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46094 dst: /127.0.0.1:34335 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741935_1117 (size=5567) 2024-11-21T12:30:56,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741935_1117 (size=5567) 2024-11-21T12:30:56,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741935_1117 (size=5567) 2024-11-21T12:30:58,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 after 4002ms 2024-11-21T12:30:58,319 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:58,319 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 took 4031ms 2024-11-21T12:30:58,321 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062; continuing. 2024-11-21T12:30:58,321 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 so closing down 2024-11-21T12:30:58,321 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T12:30:58,323 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732192254062.temp 2024-11-21T12:30:58,350 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp 2024-11-21T12:30:58,350 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T12:30:58,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741938_1118 (size=5575) 2024-11-21T12:30:58,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741938_1118 (size=5575) 2024-11-21T12:30:58,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741938_1118 (size=5575) 2024-11-21T12:30:58,357 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-21T12:30:58,359 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 2024-11-21T12:30:58,359 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, size=0, length=0, corrupted=false, cancelled=false 2024-11-21T12:30:58,359 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, journal: Splitting hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, size=0 (0bytes) at 1732192254288Finishing writing output for hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 so closing down at 1732192258321 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp at 1732192258350 (+29 ms)3 split writer threads finished at 1732192258350Closed recovered edits writer path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732192258357 (+7 ms)Rename recovered edits hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000037-wal.1732192254062.temp to hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 at 1732192258359 (+2 ms)Processed 30 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062, size=0, length=0, corrupted=false, cancelled=false at 1732192258359 2024-11-21T12:30:58,360 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 to hdfs://localhost:41319/hbase/oldWALs/wal.1732192254062 2024-11-21T12:30:58,361 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 2024-11-21T12:30:58,361 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-21T12:30:58,363 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41319/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765, archiveDir=hdfs://localhost:41319/hbase/oldWALs, maxLogs=32 2024-11-21T12:30:58,376 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192258363, exclude list is [], retry=0 2024-11-21T12:30:58,378 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39871,DS-f2e1f933-ed62-4a43-b8f6-ccf81a369b28,DISK] 2024-11-21T12:30:58,378 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37507,DS-1978c4d5-2a14-43fd-9e9a-ab1ed323f959,DISK] 2024-11-21T12:30:58,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34335,DS-994861ff-3fbd-4017-8303-85affa12688c,DISK] 2024-11-21T12:30:58,406 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192258363 2024-11-21T12:30:58,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44529:44529),(127.0.0.1/127.0.0.1:33699:33699),(127.0.0.1/127.0.0.1:33707:33707)] 2024-11-21T12:30:58,407 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T12:30:58,408 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,409 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName a 2024-11-21T12:30:58,409 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:58,414 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/16e16ba4c759459f906ac35c0a1b1915 2024-11-21T12:30:58,414 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:58,414 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,415 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName b 2024-11-21T12:30:58,415 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:58,419 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/b/2e630fe646f14bdabc1faad85d2a8e43 2024-11-21T12:30:58,419 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:58,419 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,420 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbb3fad630b08d16eeb6e185c159b392 columnFamilyName c 2024-11-21T12:30:58,420 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T12:30:58,424 DEBUG [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/c/409a03d8a2984db180350cea01d9b52d 2024-11-21T12:30:58,424 INFO [StoreOpener-cbb3fad630b08d16eeb6e185c159b392-1 {}] regionserver.HStore(327): Store=cbb3fad630b08d16eeb6e185c159b392/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T12:30:58,425 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,425 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,426 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,427 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 2024-11-21T12:30:58,454 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-21T12:30:58,459 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 2024-11-21T12:30:58,459 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cbb3fad630b08d16eeb6e185c159b392 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-21T12:30:58,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/a88189f1030647dfb60a6da083d65dc1 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732192254230/Put/seqid=0 2024-11-21T12:30:58,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741940_1120 (size=5958) 2024-11-21T12:30:58,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741940_1120 (size=5958) 2024-11-21T12:30:58,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741940_1120 (size=5958) 2024-11-21T12:30:58,481 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/a88189f1030647dfb60a6da083d65dc1 2024-11-21T12:30:58,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/98c123c257324c9fba50abbfec9d4c2c is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732192254237/Put/seqid=0 2024-11-21T12:30:58,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741941_1121 (size=5958) 2024-11-21T12:30:58,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741941_1121 (size=5958) 2024-11-21T12:30:58,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741941_1121 (size=5958) 2024-11-21T12:30:58,508 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/98c123c257324c9fba50abbfec9d4c2c 2024-11-21T12:30:58,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/19fc546f88254a4da7ad976aa41cea86 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732192254243/Put/seqid=0 2024-11-21T12:30:58,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741942_1122 (size=5958) 2024-11-21T12:30:58,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741942_1122 (size=5958) 2024-11-21T12:30:58,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741942_1122 (size=5958) 2024-11-21T12:30:58,534 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/19fc546f88254a4da7ad976aa41cea86 2024-11-21T12:30:58,539 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/a/a88189f1030647dfb60a6da083d65dc1 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/a88189f1030647dfb60a6da083d65dc1 2024-11-21T12:30:58,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/a/a88189f1030647dfb60a6da083d65dc1, entries=10, sequenceid=66, filesize=5.8 K 2024-11-21T12:30:58,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/b/98c123c257324c9fba50abbfec9d4c2c as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/b/98c123c257324c9fba50abbfec9d4c2c 2024-11-21T12:30:58,547 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/b/98c123c257324c9fba50abbfec9d4c2c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-21T12:30:58,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/.tmp/c/19fc546f88254a4da7ad976aa41cea86 as hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/c/19fc546f88254a4da7ad976aa41cea86 2024-11-21T12:30:58,552 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/c/19fc546f88254a4da7ad976aa41cea86, entries=10, sequenceid=66, filesize=5.8 K 2024-11-21T12:30:58,552 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for cbb3fad630b08d16eeb6e185c159b392 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-11-21T12:30:58,553 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/0000000000000000066 2024-11-21T12:30:58,553 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,554 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,554 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-21T12:30:58,555 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cbb3fad630b08d16eeb6e185c159b392 2024-11-21T12:30:58,557 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/hbase/data/default/testReplayEditsWrittenViaHRegion/cbb3fad630b08d16eeb6e185c159b392/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-21T12:30:58,558 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cbb3fad630b08d16eeb6e185c159b392; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59861308, jitterRate=-0.10799700021743774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-21T12:30:58,558 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cbb3fad630b08d16eeb6e185c159b392: Writing region info on filesystem at 1732192258407Initializing all the Stores at 1732192258408 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192258408Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192258408Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732192258408Obtaining lock to block concurrent updates at 1732192258459 (+51 ms)Preparing flush snapshotting stores in cbb3fad630b08d16eeb6e185c159b392 at 1732192258459Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732192258459Flushing stores of testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. at 1732192258459Flushing cbb3fad630b08d16eeb6e185c159b392/a: creating writer at 1732192258459Flushing cbb3fad630b08d16eeb6e185c159b392/a: appending metadata at 1732192258474 (+15 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/a: closing flushed file at 1732192258474Flushing cbb3fad630b08d16eeb6e185c159b392/b: creating writer at 1732192258486 (+12 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/b: appending metadata at 1732192258501 (+15 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/b: closing flushed file at 1732192258501Flushing cbb3fad630b08d16eeb6e185c159b392/c: creating writer at 1732192258513 (+12 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/c: appending metadata at 1732192258527 (+14 ms)Flushing cbb3fad630b08d16eeb6e185c159b392/c: closing flushed file at 1732192258527Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@84f45c8: reopening flushed file at 1732192258538 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1858ba45: reopening flushed file at 1732192258543 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@521fa8e3: reopening flushed file at 1732192258547 (+4 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for cbb3fad630b08d16eeb6e185c159b392 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1732192258552 (+5 ms)Cleaning up temporary data from old regions at 1732192258554 (+2 ms)Region opened successfully at 1732192258558 (+4 ms) 2024-11-21T12:30:58,571 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cbb3fad630b08d16eeb6e185c159b392, disabling compactions & flushes 2024-11-21T12:30:58,571 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:58,571 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:58,571 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. after waiting 0 ms 2024-11-21T12:30:58,571 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:58,572 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732192253767.cbb3fad630b08d16eeb6e185c159b392. 2024-11-21T12:30:58,572 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cbb3fad630b08d16eeb6e185c159b392: Waiting for close lock at 1732192258571Disabling compacts and flushes for region at 1732192258571Disabling writes for close at 1732192258571Writing region close event to WAL at 1732192258572 (+1 ms)Closed at 1732192258572 2024-11-21T12:30:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741939_1119 (size=175) 2024-11-21T12:30:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741939_1119 (size=175) 2024-11-21T12:30:58,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741939_1119 (size=175) 2024-11-21T12:30:58,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-21T12:30:58,577 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732192258363) 2024-11-21T12:30:58,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 442) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41319 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1932268574_22 at /127.0.0.1:45882 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1932268574_22 at /127.0.0.1:36968 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1932268574_22 at /127.0.0.1:49204 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41319 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=6939 (was 6946) 2024-11-21T12:30:58,592 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-21T12:30:58,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T12:30:58,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T12:30:58,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T12:30:58,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T12:30:58,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T12:30:58,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1962663776, stopped=false 2024-11-21T12:30:58,593 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9bb5b7e9dd2c,32899,1732192215752 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:58,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T12:30:58,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:58,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T12:30:58,596 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T12:30:58,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:58,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:58,597 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T12:30:58,597 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9bb5b7e9dd2c,36317,1732192216659' ***** 2024-11-21T12:30:58,597 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T12:30:58,597 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9bb5b7e9dd2c,44681,1732192216704' ***** 2024-11-21T12:30:58,597 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T12:30:58,597 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T12:30:58,597 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T12:30:58,597 INFO [RS:1;9bb5b7e9dd2c:36317 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T12:30:58,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T12:30:58,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T12:30:58,597 INFO [RS:2;9bb5b7e9dd2c:44681 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T12:30:58,598 INFO [RS:2;9bb5b7e9dd2c:44681 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T12:30:58,598 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(959): stopping server 9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:58,598 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T12:30:58,598 INFO [RS:2;9bb5b7e9dd2c:44681 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9bb5b7e9dd2c:44681. 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:58,598 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T12:30:58,598 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(959): stopping server 9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:58,598 INFO [regionserver/9bb5b7e9dd2c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:58,598 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(976): stopping server 9bb5b7e9dd2c,44681,1732192216704; all regions closed. 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9bb5b7e9dd2c:36317. 2024-11-21T12:30:58,598 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ca0f0d87cbf2161be78e638a1338fb7, disabling compactions & flushes 2024-11-21T12:30:58,598 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T12:30:58,598 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:58,598 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,598 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:58,598 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. after waiting 0 ms 2024-11-21T12:30:58,598 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T12:30:58,598 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T12:30:58,599 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T12:30:58,599 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T12:30:58,599 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6ca0f0d87cbf2161be78e638a1338fb7=testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7.} 2024-11-21T12:30:58,599 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6ca0f0d87cbf2161be78e638a1338fb7 2024-11-21T12:30:58,599 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T12:30:58,599 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T12:30:58,599 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T12:30:58,599 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T12:30:58,599 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T12:30:58,599 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-21T12:30:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741835_1011 (size=175) 2024-11-21T12:30:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741835_1011 (size=175) 2024-11-21T12:30:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741835_1011 (size=175) 2024-11-21T12:30:58,605 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs 2024-11-21T12:30:58,605 INFO [RS:2;9bb5b7e9dd2c:44681 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9bb5b7e9dd2c%2C44681%2C1732192216704:(num 1732192218415) 2024-11-21T12:30:58,605 DEBUG [RS:2;9bb5b7e9dd2c:44681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,605 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:58,605 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.ChoreService(370): Chore service for: regionserver/9bb5b7e9dd2c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T12:30:58,606 INFO [regionserver/9bb5b7e9dd2c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T12:30:58,606 INFO [RS:2;9bb5b7e9dd2c:44681 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44681 2024-11-21T12:30:58,607 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/cf7e54a360cd4a3386e0dd448aa5beb2 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7./info:regioninfo/1732192237323/Put/seqid=0 2024-11-21T12:30:58,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9bb5b7e9dd2c,44681,1732192216704 2024-11-21T12:30:58,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T12:30:58,608 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T12:30:58,609 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ca0f0d87cbf2161be78e638a1338fb7/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-21T12:30:58,609 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9bb5b7e9dd2c,44681,1732192216704] 2024-11-21T12:30:58,610 INFO [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:58,610 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ca0f0d87cbf2161be78e638a1338fb7: Waiting for close lock at 1732192258598Running coprocessor pre-close hooks at 1732192258598Disabling compacts and flushes for region at 1732192258598Disabling writes for close at 1732192258598Writing region close event to WAL at 1732192258600 (+2 ms)Running coprocessor post-close hooks at 1732192258610 (+10 ms)Closed at 1732192258610 2024-11-21T12:30:58,610 DEBUG [RS_CLOSE_REGION-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732192233140.6ca0f0d87cbf2161be78e638a1338fb7. 2024-11-21T12:30:58,611 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9bb5b7e9dd2c,44681,1732192216704 already deleted, retry=false 2024-11-21T12:30:58,611 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9bb5b7e9dd2c,44681,1732192216704 expired; onlineServers=1 2024-11-21T12:30:58,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741943_1123 (size=6778) 2024-11-21T12:30:58,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741943_1123 (size=6778) 2024-11-21T12:30:58,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741943_1123 (size=6778) 2024-11-21T12:30:58,619 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/cf7e54a360cd4a3386e0dd448aa5beb2 2024-11-21T12:30:58,626 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/.tmp/info/cf7e54a360cd4a3386e0dd448aa5beb2 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/info/cf7e54a360cd4a3386e0dd448aa5beb2 2024-11-21T12:30:58,631 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/info/cf7e54a360cd4a3386e0dd448aa5beb2, entries=8, sequenceid=23, filesize=6.6 K 2024-11-21T12:30:58,632 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=23, compaction requested=false 2024-11-21T12:30:58,636 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-21T12:30:58,637 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T12:30:58,637 INFO [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T12:30:58,637 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732192258599Running coprocessor pre-close hooks at 1732192258599Disabling compacts and flushes for region at 1732192258599Disabling writes for close at 1732192258599Obtaining lock to block concurrent updates at 1732192258599Preparing flush snapshotting stores in 1588230740 at 1732192258599Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732192258600 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732192258602 (+2 ms)Flushing 1588230740/info: creating writer at 1732192258602Flushing 1588230740/info: appending metadata at 1732192258606 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732192258606Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b5cf12: reopening flushed file at 1732192258625 (+19 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=23, compaction requested=false at 1732192258632 (+7 ms)Writing region close event to WAL at 1732192258633 (+1 ms)Running coprocessor post-close hooks at 1732192258637 (+4 ms)Closed at 1732192258637 2024-11-21T12:30:58,637 DEBUG [RS_CLOSE_META-regionserver/9bb5b7e9dd2c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T12:30:58,664 INFO [regionserver/9bb5b7e9dd2c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:58,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:58,710 INFO [RS:2;9bb5b7e9dd2c:44681 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T12:30:58,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44681-0x1006251a9c40003, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:58,710 INFO [RS:2;9bb5b7e9dd2c:44681 {}] regionserver.HRegionServer(1031): Exiting; stopping=9bb5b7e9dd2c,44681,1732192216704; zookeeper connection closed. 2024-11-21T12:30:58,710 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c43c474 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c43c474 2024-11-21T12:30:58,799 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(976): stopping server 9bb5b7e9dd2c,36317,1732192216659; all regions closed. 2024-11-21T12:30:58,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741893_1071 (size=2625) 2024-11-21T12:30:58,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741893_1071 (size=2625) 2024-11-21T12:30:58,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741893_1071 (size=2625) 2024-11-21T12:30:58,805 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs 2024-11-21T12:30:58,805 INFO [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9bb5b7e9dd2c%2C36317%2C1732192216659.meta:.meta(num 1732192236389) 2024-11-21T12:30:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741833_1009 (size=3621) 2024-11-21T12:30:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741833_1009 (size=3621) 2024-11-21T12:30:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741833_1009 (size=3621) 2024-11-21T12:30:58,809 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/oldWALs 2024-11-21T12:30:58,809 INFO [RS:1;9bb5b7e9dd2c:36317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9bb5b7e9dd2c%2C36317%2C1732192216659:(num 1732192218415) 2024-11-21T12:30:58,810 DEBUG [RS:1;9bb5b7e9dd2c:36317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T12:30:58,810 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T12:30:58,810 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T12:30:58,810 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.ChoreService(370): Chore service for: regionserver/9bb5b7e9dd2c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T12:30:58,810 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T12:30:58,810 INFO [regionserver/9bb5b7e9dd2c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T12:30:58,810 INFO [RS:1;9bb5b7e9dd2c:36317 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36317 2024-11-21T12:30:58,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9bb5b7e9dd2c,36317,1732192216659 2024-11-21T12:30:58,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T12:30:58,812 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T12:30:58,813 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9bb5b7e9dd2c,36317,1732192216659] 2024-11-21T12:30:58,815 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9bb5b7e9dd2c,36317,1732192216659 already deleted, retry=false 2024-11-21T12:30:58,815 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9bb5b7e9dd2c,36317,1732192216659 expired; onlineServers=0 2024-11-21T12:30:58,815 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9bb5b7e9dd2c,32899,1732192215752' ***** 2024-11-21T12:30:58,815 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T12:30:58,815 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T12:30:58,815 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T12:30:58,815 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T12:30:58,815 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T12:30:58,816 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T12:30:58,816 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.large.0-1732192217985 {}] cleaner.HFileCleaner(306): Exit Thread[master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.large.0-1732192217985,5,FailOnTimeoutGroup] 2024-11-21T12:30:58,816 DEBUG [master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.small.0-1732192217986 {}] cleaner.HFileCleaner(306): Exit Thread[master/9bb5b7e9dd2c:0:becomeActiveMaster-HFileCleaner.small.0-1732192217986,5,FailOnTimeoutGroup] 2024-11-21T12:30:58,816 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.ChoreService(370): Chore service for: master/9bb5b7e9dd2c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T12:30:58,816 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T12:30:58,816 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] master.HMaster(1795): Stopping service threads 2024-11-21T12:30:58,816 INFO [M:0;9bb5b7e9dd2c:32899 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T12:30:58,816 INFO [M:0;9bb5b7e9dd2c:32899 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T12:30:58,817 INFO [M:0;9bb5b7e9dd2c:32899 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T12:30:58,817 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T12:30:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T12:30:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T12:30:58,818 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] zookeeper.ZKUtil(347): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T12:30:58,818 WARN [M:0;9bb5b7e9dd2c:32899 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T12:30:58,819 INFO [M:0;9bb5b7e9dd2c:32899 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/.lastflushedseqids 2024-11-21T12:30:58,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741944_1124 (size=119) 2024-11-21T12:30:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741944_1124 (size=119) 2024-11-21T12:30:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741944_1124 (size=119) 2024-11-21T12:30:58,837 INFO [M:0;9bb5b7e9dd2c:32899 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T12:30:58,837 INFO [M:0;9bb5b7e9dd2c:32899 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T12:30:58,837 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T12:30:58,837 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:58,837 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:58,837 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T12:30:58,837 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:58,837 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-21T12:30:58,853 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca390c7c1da04ae88bb66f21c635c76c is 82, key is hbase:meta,,1/info:regioninfo/1732192236626/Put/seqid=0 2024-11-21T12:30:58,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741945_1125 (size=6063) 2024-11-21T12:30:58,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741945_1125 (size=6063) 2024-11-21T12:30:58,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741945_1125 (size=6063) 2024-11-21T12:30:58,859 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca390c7c1da04ae88bb66f21c635c76c 2024-11-21T12:30:58,880 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97a0682266514bd6ab0117e605a96b16 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732192233587/Put/seqid=0 2024-11-21T12:30:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741946_1126 (size=7907) 2024-11-21T12:30:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741946_1126 (size=7907) 2024-11-21T12:30:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741946_1126 (size=7907) 2024-11-21T12:30:58,887 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97a0682266514bd6ab0117e605a96b16 2024-11-21T12:30:58,891 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97a0682266514bd6ab0117e605a96b16 2024-11-21T12:30:58,909 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/26043ef8734d4aefa756fcae1a36620e is 69, key is 9bb5b7e9dd2c,36317,1732192216659/rs:state/1732192218114/Put/seqid=0 2024-11-21T12:30:58,914 INFO [RS:1;9bb5b7e9dd2c:36317 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T12:30:58,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:58,914 INFO [RS:1;9bb5b7e9dd2c:36317 {}] regionserver.HRegionServer(1031): Exiting; stopping=9bb5b7e9dd2c,36317,1732192216659; zookeeper connection closed. 2024-11-21T12:30:58,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36317-0x1006251a9c40002, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:58,915 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10c306c3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10c306c3 2024-11-21T12:30:58,915 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T12:30:58,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741947_1127 (size=5445) 2024-11-21T12:30:58,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741947_1127 (size=5445) 2024-11-21T12:30:58,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741947_1127 (size=5445) 2024-11-21T12:30:58,917 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/26043ef8734d4aefa756fcae1a36620e 2024-11-21T12:30:58,922 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26043ef8734d4aefa756fcae1a36620e 2024-11-21T12:30:58,923 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ca390c7c1da04ae88bb66f21c635c76c as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ca390c7c1da04ae88bb66f21c635c76c 2024-11-21T12:30:58,928 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ca390c7c1da04ae88bb66f21c635c76c, entries=14, sequenceid=207, filesize=5.9 K 2024-11-21T12:30:58,929 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97a0682266514bd6ab0117e605a96b16 as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97a0682266514bd6ab0117e605a96b16 2024-11-21T12:30:58,934 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97a0682266514bd6ab0117e605a96b16 2024-11-21T12:30:58,934 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97a0682266514bd6ab0117e605a96b16, entries=21, sequenceid=207, filesize=7.7 K 2024-11-21T12:30:58,935 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/26043ef8734d4aefa756fcae1a36620e as hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/26043ef8734d4aefa756fcae1a36620e 2024-11-21T12:30:58,940 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26043ef8734d4aefa756fcae1a36620e 2024-11-21T12:30:58,940 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41319/user/jenkins/test-data/d9b0cd82-f600-1348-9708-1abf219d2cc6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/26043ef8734d4aefa756fcae1a36620e, entries=3, sequenceid=207, filesize=5.3 K 2024-11-21T12:30:58,941 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=207, compaction requested=false 2024-11-21T12:30:58,942 INFO [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T12:30:58,943 DEBUG [M:0;9bb5b7e9dd2c:32899 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732192258837Disabling compacts and flushes for region at 1732192258837Disabling writes for close at 1732192258837Obtaining lock to block concurrent updates at 1732192258837Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732192258837Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732192258838 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732192258838Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732192258838Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732192258853 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732192258853Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732192258864 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732192258879 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732192258879Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732192258891 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732192258908 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732192258908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a9ee942: reopening flushed file at 1732192258922 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93346c0: reopening flushed file at 1732192258929 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60452b88: reopening flushed file at 1732192258934 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=207, compaction requested=false at 1732192258941 (+7 ms)Writing region close event to WAL at 1732192258942 (+1 ms)Closed at 1732192258942 2024-11-21T12:30:58,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37507 is added to blk_1073741830_1006 (size=102162) 2024-11-21T12:30:58,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34335 is added to blk_1073741830_1006 (size=102162) 2024-11-21T12:30:58,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39871 is added to blk_1073741830_1006 (size=102162) 2024-11-21T12:30:58,946 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T12:30:58,946 INFO [M:0;9bb5b7e9dd2c:32899 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T12:30:58,946 INFO [M:0;9bb5b7e9dd2c:32899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32899 2024-11-21T12:30:58,946 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T12:30:59,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:59,048 INFO [M:0;9bb5b7e9dd2c:32899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T12:30:59,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32899-0x1006251a9c40000, quorum=127.0.0.1:55541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T12:30:59,054 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192247995 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192247995 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732192238634/wal.1732192247995 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-21T12:30:59,057 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192238413 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192238413 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732192237856/wal.1732192238413 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-21T12:30:59,058 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732192228531/wal.1732192228667 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:59,060 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248913 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248913 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732192248296/wal.1732192248913 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-21T12:30:59,061 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732192249141/wal.1732192249232 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:59,061 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732192253765/wal.1732192254062 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:59,061 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732192221322/wal.1732192221429 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T12:30:59,063 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal.1732192221107 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal.1732192221107 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732192220249/wal.1732192221107 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-21T12:30:59,065 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal.1732192219898 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal.1732192219898 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732192219356/wal.1732192219898 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1519374329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-21T12:30:59,069 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:59,071 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T12:30:59,072 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T12:30:59,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T12:30:59,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,STOPPED} 2024-11-21T12:30:59,075 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T12:30:59,075 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T12:30:59,075 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1168079295-172.17.0.2-1732192212418 (Datanode Uuid 411fc6f6-12a3-407f-805e-27aee14c66de) service to localhost/127.0.0.1:41319 2024-11-21T12:30:59,075 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T12:30:59,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data5/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data6/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,077 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T12:30:59,079 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:59,079 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T12:30:59,079 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T12:30:59,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T12:30:59,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,STOPPED} 2024-11-21T12:30:59,081 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T12:30:59,081 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T12:30:59,081 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1168079295-172.17.0.2-1732192212418 (Datanode Uuid f7762d60-0d12-48a7-a3e4-9ddf77859730) service to localhost/127.0.0.1:41319 2024-11-21T12:30:59,081 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T12:30:59,082 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data3/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,082 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data4/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,082 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T12:30:59,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T12:30:59,085 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T12:30:59,086 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T12:30:59,086 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T12:30:59,086 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,STOPPED} 2024-11-21T12:30:59,087 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T12:30:59,087 WARN [BP-1168079295-172.17.0.2-1732192212418 heartbeating to localhost/127.0.0.1:41319 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1168079295-172.17.0.2-1732192212418 (Datanode Uuid ae61627a-96b8-446d-bd16-0c59bc3292ba) service to localhost/127.0.0.1:41319 2024-11-21T12:30:59,087 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T12:30:59,087 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T12:30:59,088 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data1/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,088 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/cluster_dea9ce1c-215d-cd01-3258-5f05ba6a7fbd/data/data2/current/BP-1168079295-172.17.0.2-1732192212418 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T12:30:59,088 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T12:30:59,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T12:30:59,095 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T12:30:59,095 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T12:30:59,096 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T12:30:59,096 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1560bbe-4753-82a6-e10f-6be002a440a2/hadoop.log.dir/,STOPPED} 2024-11-21T12:30:59,106 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T12:30:59,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down