2024-11-20 22:34:30,112 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 22:34:30,128 main DEBUG Took 0.012882 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 22:34:30,128 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 22:34:30,129 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 22:34:30,130 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 22:34:30,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,139 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 22:34:30,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,159 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,161 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,162 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,164 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,166 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,168 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,168 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,169 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,170 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,171 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 22:34:30,172 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,172 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 22:34:30,174 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 22:34:30,176 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 22:34:30,178 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 22:34:30,179 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 22:34:30,180 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 22:34:30,181 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 22:34:30,190 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 22:34:30,194 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 22:34:30,196 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 22:34:30,196 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 22:34:30,197 main DEBUG createAppenders(={Console}) 2024-11-20 22:34:30,198 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-20 22:34:30,198 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 22:34:30,198 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-20 22:34:30,199 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 22:34:30,199 main DEBUG OutputStream closed 2024-11-20 22:34:30,199 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 22:34:30,200 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 22:34:30,200 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-20 22:34:30,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 22:34:30,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 22:34:30,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 22:34:30,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 22:34:30,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 22:34:30,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 22:34:30,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 22:34:30,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 22:34:30,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 22:34:30,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 22:34:30,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 22:34:30,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 22:34:30,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 22:34:30,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 22:34:30,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 22:34:30,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 22:34:30,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 22:34:30,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 22:34:30,278 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 22:34:30,278 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-20 22:34:30,278 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 22:34:30,279 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-20T22:34:30,605 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80 2024-11-20 22:34:30,608 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 22:34:30,609 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T22:34:30,622 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-20T22:34:30,631 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-20T22:34:30,662 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T22:34:30,732 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T22:34:30,733 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T22:34:30,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T22:34:30,778 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947, deleteOnExit=true 2024-11-20T22:34:30,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T22:34:30,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/test.cache.data in system properties and HBase conf 2024-11-20T22:34:30,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T22:34:30,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir in system properties and HBase conf 2024-11-20T22:34:30,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T22:34:30,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T22:34:30,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T22:34:30,894 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T22:34:31,008 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T22:34:31,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T22:34:31,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T22:34:31,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T22:34:31,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T22:34:31,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T22:34:31,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T22:34:31,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T22:34:31,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T22:34:31,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T22:34:31,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/nfs.dump.dir in system properties and HBase conf 2024-11-20T22:34:31,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/java.io.tmpdir in system properties and HBase conf 2024-11-20T22:34:31,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T22:34:31,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T22:34:31,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T22:34:32,135 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T22:34:32,216 INFO [Time-limited test {}] log.Log(170): Logging initialized @2900ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T22:34:32,298 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T22:34:32,394 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T22:34:32,423 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T22:34:32,424 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T22:34:32,426 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T22:34:32,447 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T22:34:32,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,AVAILABLE} 2024-11-20T22:34:32,457 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T22:34:32,682 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/java.io.tmpdir/jetty-localhost-38573-hadoop-hdfs-3_4_1-tests_jar-_-any-8657307405380772124/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T22:34:32,694 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:38573} 2024-11-20T22:34:32,695 INFO [Time-limited test {}] server.Server(415): Started @3379ms 2024-11-20T22:34:33,329 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T22:34:33,340 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T22:34:33,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T22:34:33,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T22:34:33,342 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T22:34:33,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,AVAILABLE} 2024-11-20T22:34:33,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T22:34:33,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/java.io.tmpdir/jetty-localhost-33139-hadoop-hdfs-3_4_1-tests_jar-_-any-17492916977889451449/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:34:33,456 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:33139} 2024-11-20T22:34:33,456 INFO [Time-limited test {}] server.Server(415): Started @4141ms 2024-11-20T22:34:33,511 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T22:34:33,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T22:34:33,678 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T22:34:33,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T22:34:33,691 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T22:34:33,692 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T22:34:33,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,AVAILABLE} 2024-11-20T22:34:33,697 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T22:34:33,833 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/java.io.tmpdir/jetty-localhost-38975-hadoop-hdfs-3_4_1-tests_jar-_-any-4594436449351852956/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:34:33,835 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:38975} 2024-11-20T22:34:33,835 INFO [Time-limited test {}] server.Server(415): Started @4520ms 2024-11-20T22:34:33,839 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T22:34:33,901 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T22:34:33,908 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T22:34:33,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T22:34:33,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T22:34:33,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T22:34:33,919 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,AVAILABLE} 2024-11-20T22:34:33,920 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T22:34:34,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/java.io.tmpdir/jetty-localhost-45701-hadoop-hdfs-3_4_1-tests_jar-_-any-17952561689703175394/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:34:34,066 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:45701} 2024-11-20T22:34:34,066 INFO [Time-limited test {}] server.Server(415): Started @4751ms 2024-11-20T22:34:34,069 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T22:34:35,412 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data4/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,412 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data3/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,412 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data1/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,412 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data2/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,446 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T22:34:35,446 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T22:34:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb775b2793fb4faff with lease ID 0xb6e4434e628056c3: Processing first storage report for DS-f243a7d1-9057-4c43-8520-c365588218cf from datanode DatanodeRegistration(127.0.0.1:36643, datanodeUuid=201b5eee-3952-42bb-873b-5a45722d3c69, infoPort=36477, infoSecurePort=0, ipcPort=46117, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,493 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb775b2793fb4faff with lease ID 0xb6e4434e628056c3: from storage DS-f243a7d1-9057-4c43-8520-c365588218cf node DatanodeRegistration(127.0.0.1:36643, datanodeUuid=201b5eee-3952-42bb-873b-5a45722d3c69, infoPort=36477, infoSecurePort=0, ipcPort=46117, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,493 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6557c1c37f76e13c with lease ID 0xb6e4434e628056c2: Processing first storage report for DS-0be96184-fe36-4f1d-8e95-83bd78555f17 from datanode DatanodeRegistration(127.0.0.1:45551, datanodeUuid=e28c0d50-94dc-4ddc-8035-8a0aa1b81890, infoPort=34173, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6557c1c37f76e13c with lease ID 0xb6e4434e628056c2: from storage DS-0be96184-fe36-4f1d-8e95-83bd78555f17 node DatanodeRegistration(127.0.0.1:45551, datanodeUuid=e28c0d50-94dc-4ddc-8035-8a0aa1b81890, infoPort=34173, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,494 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data5/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,494 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data6/current/BP-1724098941-172.17.0.2-1732142071568/current, will proceed with Du for space computation calculation, 2024-11-20T22:34:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb775b2793fb4faff with lease ID 0xb6e4434e628056c3: Processing first storage report for DS-8c1f0fa0-0d65-43a3-903b-db312622276f from datanode DatanodeRegistration(127.0.0.1:36643, datanodeUuid=201b5eee-3952-42bb-873b-5a45722d3c69, infoPort=36477, infoSecurePort=0, ipcPort=46117, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb775b2793fb4faff with lease ID 0xb6e4434e628056c3: from storage DS-8c1f0fa0-0d65-43a3-903b-db312622276f node DatanodeRegistration(127.0.0.1:36643, datanodeUuid=201b5eee-3952-42bb-873b-5a45722d3c69, infoPort=36477, infoSecurePort=0, ipcPort=46117, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6557c1c37f76e13c with lease ID 0xb6e4434e628056c2: Processing first storage report for DS-db704751-dee6-444a-a291-8d7111fe9872 from datanode DatanodeRegistration(127.0.0.1:45551, datanodeUuid=e28c0d50-94dc-4ddc-8035-8a0aa1b81890, infoPort=34173, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6557c1c37f76e13c with lease ID 0xb6e4434e628056c2: from storage DS-db704751-dee6-444a-a291-8d7111fe9872 node DatanodeRegistration(127.0.0.1:45551, datanodeUuid=e28c0d50-94dc-4ddc-8035-8a0aa1b81890, infoPort=34173, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,516 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T22:34:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd22031ac1869a0cf with lease ID 0xb6e4434e628056c4: Processing first storage report for DS-cef45184-81f1-4e78-82eb-3788ee684bc1 from datanode DatanodeRegistration(127.0.0.1:37697, datanodeUuid=d6591d0a-a4f6-4a72-8c20-0bf3d2fc991d, infoPort=39567, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd22031ac1869a0cf with lease ID 0xb6e4434e628056c4: from storage DS-cef45184-81f1-4e78-82eb-3788ee684bc1 node DatanodeRegistration(127.0.0.1:37697, datanodeUuid=d6591d0a-a4f6-4a72-8c20-0bf3d2fc991d, infoPort=39567, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd22031ac1869a0cf with lease ID 0xb6e4434e628056c4: Processing first storage report for DS-b5949852-c691-4dcd-952d-4e664c118a68 from datanode DatanodeRegistration(127.0.0.1:37697, datanodeUuid=d6591d0a-a4f6-4a72-8c20-0bf3d2fc991d, infoPort=39567, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568) 2024-11-20T22:34:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd22031ac1869a0cf with lease ID 0xb6e4434e628056c4: from storage DS-b5949852-c691-4dcd-952d-4e664c118a68 node DatanodeRegistration(127.0.0.1:37697, datanodeUuid=d6591d0a-a4f6-4a72-8c20-0bf3d2fc991d, infoPort=39567, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=456250372;c=1732142071568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T22:34:35,618 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80 2024-11-20T22:34:35,687 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/zookeeper_0, clientPort=62486, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T22:34:35,728 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62486 2024-11-20T22:34:35,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:35,747 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:35,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741825_1001 (size=7) 2024-11-20T22:34:35,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741825_1001 (size=7) 2024-11-20T22:34:35,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741825_1001 (size=7) 2024-11-20T22:34:36,352 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 with version=8 2024-11-20T22:34:36,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/hbase-staging 2024-11-20T22:34:36,658 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9f31b4695a16:0 server-side Connection retries=45 2024-11-20T22:34:36,666 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:36,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:36,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T22:34:36,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:36,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T22:34:36,800 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T22:34:36,853 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T22:34:36,860 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T22:34:36,864 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T22:34:36,887 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92080 (auto-detected) 2024-11-20T22:34:36,888 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T22:34:36,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40769 2024-11-20T22:34:36,928 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40769 connecting to ZooKeeper ensemble=127.0.0.1:62486 2024-11-20T22:34:37,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:407690x0, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T22:34:37,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40769-0x1015a628ac10000 connected 2024-11-20T22:34:37,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,169 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,181 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:34:37,186 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647, hbase.cluster.distributed=false 2024-11-20T22:34:37,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T22:34:37,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-20T22:34:37,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40769 2024-11-20T22:34:37,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40769 2024-11-20T22:34:37,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-20T22:34:37,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-20T22:34:37,380 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9f31b4695a16:0 server-side Connection retries=45 2024-11-20T22:34:37,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,383 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T22:34:37,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T22:34:37,386 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T22:34:37,389 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T22:34:37,391 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43093 2024-11-20T22:34:37,394 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43093 connecting to ZooKeeper ensemble=127.0.0.1:62486 2024-11-20T22:34:37,395 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430930x0, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T22:34:37,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43093-0x1015a628ac10001 connected 2024-11-20T22:34:37,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:34:37,434 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T22:34:37,444 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T22:34:37,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T22:34:37,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T22:34:37,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43093 2024-11-20T22:34:37,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43093 2024-11-20T22:34:37,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43093 2024-11-20T22:34:37,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43093 2024-11-20T22:34:37,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43093 2024-11-20T22:34:37,481 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9f31b4695a16:0 server-side Connection retries=45 2024-11-20T22:34:37,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,482 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T22:34:37,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T22:34:37,483 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T22:34:37,483 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T22:34:37,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43815 2024-11-20T22:34:37,487 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43815 connecting to ZooKeeper ensemble=127.0.0.1:62486 2024-11-20T22:34:37,489 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438150x0, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T22:34:37,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438150x0, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:34:37,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43815-0x1015a628ac10002 connected 2024-11-20T22:34:37,511 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T22:34:37,514 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T22:34:37,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T22:34:37,517 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T22:34:37,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-20T22:34:37,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43815 2024-11-20T22:34:37,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43815 2024-11-20T22:34:37,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-20T22:34:37,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43815 2024-11-20T22:34:37,540 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9f31b4695a16:0 server-side Connection retries=45 2024-11-20T22:34:37,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,541 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T22:34:37,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T22:34:37,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T22:34:37,542 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T22:34:37,542 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T22:34:37,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36989 2024-11-20T22:34:37,545 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36989 connecting to ZooKeeper ensemble=127.0.0.1:62486 2024-11-20T22:34:37,546 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,549 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369890x0, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T22:34:37,563 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369890x0, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:34:37,563 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36989-0x1015a628ac10003 connected 2024-11-20T22:34:37,563 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T22:34:37,564 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T22:34:37,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T22:34:37,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T22:34:37,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36989 2024-11-20T22:34:37,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36989 2024-11-20T22:34:37,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36989 2024-11-20T22:34:37,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36989 2024-11-20T22:34:37,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36989 2024-11-20T22:34:37,586 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9f31b4695a16:40769 2024-11-20T22:34:37,587 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9f31b4695a16,40769,1732142076492 2024-11-20T22:34:37,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,606 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9f31b4695a16,40769,1732142076492 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,647 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T22:34:37,648 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9f31b4695a16,40769,1732142076492 from backup master directory 2024-11-20T22:34:37,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9f31b4695a16,40769,1732142076492 2024-11-20T22:34:37,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T22:34:37,658 WARN [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T22:34:37,658 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9f31b4695a16,40769,1732142076492 2024-11-20T22:34:37,661 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T22:34:37,662 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T22:34:37,721 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/hbase.id] with ID: 6811c5ba-3839-4753-8916-d16cd6b50586 2024-11-20T22:34:37,721 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/.tmp/hbase.id 2024-11-20T22:34:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741826_1002 (size=42) 2024-11-20T22:34:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741826_1002 (size=42) 2024-11-20T22:34:37,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741826_1002 (size=42) 2024-11-20T22:34:37,738 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/.tmp/hbase.id]:[hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/hbase.id] 2024-11-20T22:34:37,795 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T22:34:37,799 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T22:34:37,820 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-20T22:34:37,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:37,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741827_1003 (size=196) 2024-11-20T22:34:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741827_1003 (size=196) 2024-11-20T22:34:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741827_1003 (size=196) 2024-11-20T22:34:37,899 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T22:34:37,902 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T22:34:37,910 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:37,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741828_1004 (size=1189) 2024-11-20T22:34:37,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741828_1004 (size=1189) 2024-11-20T22:34:37,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741828_1004 (size=1189) 2024-11-20T22:34:37,996 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store 2024-11-20T22:34:38,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741829_1005 (size=34) 2024-11-20T22:34:38,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741829_1005 (size=34) 2024-11-20T22:34:38,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741829_1005 (size=34) 2024-11-20T22:34:38,030 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T22:34:38,034 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:38,035 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T22:34:38,036 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:34:38,036 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:34:38,038 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T22:34:38,038 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:34:38,039 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:34:38,040 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732142078035Disabling compacts and flushes for region at 1732142078035Disabling writes for close at 1732142078038 (+3 ms)Writing region close event to WAL at 1732142078038Closed at 1732142078038 2024-11-20T22:34:38,043 WARN [master/9f31b4695a16:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/.initializing 2024-11-20T22:34:38,043 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/WALs/9f31b4695a16,40769,1732142076492 2024-11-20T22:34:38,052 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:38,069 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C40769%2C1732142076492, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/WALs/9f31b4695a16,40769,1732142076492, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/oldWALs, maxLogs=10 2024-11-20T22:34:38,098 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/WALs/9f31b4695a16,40769,1732142076492/9f31b4695a16%2C40769%2C1732142076492.1732142078074, exclude list is [], retry=0 2024-11-20T22:34:38,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:38,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:38,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:38,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:38,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T22:34:38,185 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-20T22:34:38,283 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/WALs/9f31b4695a16,40769,1732142076492/9f31b4695a16%2C40769%2C1732142076492.1732142078074 2024-11-20T22:34:38,284 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:38,285 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:38,285 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:38,289 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,290 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T22:34:38,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:38,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:38,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T22:34:38,390 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:38,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:38,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T22:34:38,399 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:38,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:38,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T22:34:38,411 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:38,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:38,416 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,421 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,422 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,432 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,433 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,437 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T22:34:38,442 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T22:34:38,448 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:38,450 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71683578, jitterRate=0.06816855072975159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T22:34:38,460 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732142078304Initializing all the Stores at 1732142078307 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142078307Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142078308 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142078309 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142078309Cleaning up temporary data from old regions at 1732142078434 (+125 ms)Region opened successfully at 1732142078460 (+26 ms) 2024-11-20T22:34:38,462 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T22:34:38,504 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7046067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f31b4695a16/172.17.0.2:0 2024-11-20T22:34:38,544 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T22:34:38,555 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T22:34:38,555 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T22:34:38,558 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T22:34:38,559 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T22:34:38,565 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T22:34:38,566 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T22:34:38,596 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T22:34:38,606 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T22:34:38,668 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T22:34:38,671 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T22:34:38,673 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T22:34:38,683 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T22:34:38,687 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T22:34:38,692 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T22:34:38,704 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T22:34:38,706 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T22:34:38,719 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T22:34:38,744 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T22:34:38,757 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T22:34:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T22:34:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T22:34:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T22:34:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T22:34:38,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,777 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9f31b4695a16,40769,1732142076492, sessionid=0x1015a628ac10000, setting cluster-up flag (Was=false) 2024-11-20T22:34:38,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,842 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T22:34:38,844 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f31b4695a16,40769,1732142076492 2024-11-20T22:34:38,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:38,898 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T22:34:38,900 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f31b4695a16,40769,1732142076492 2024-11-20T22:34:38,911 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T22:34:38,982 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(746): ClusterId : 6811c5ba-3839-4753-8916-d16cd6b50586 2024-11-20T22:34:38,983 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(746): ClusterId : 6811c5ba-3839-4753-8916-d16cd6b50586 2024-11-20T22:34:38,984 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(746): ClusterId : 6811c5ba-3839-4753-8916-d16cd6b50586 2024-11-20T22:34:38,985 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T22:34:38,985 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T22:34:38,985 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T22:34:39,013 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T22:34:39,013 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T22:34:39,014 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T22:34:39,014 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T22:34:39,014 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T22:34:39,014 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T22:34:39,014 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T22:34:39,023 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T22:34:39,030 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T22:34:39,036 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T22:34:39,036 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T22:34:39,036 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T22:34:39,037 DEBUG [RS:2;9f31b4695a16:36989 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57bd57b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f31b4695a16/172.17.0.2:0 2024-11-20T22:34:39,037 DEBUG [RS:0;9f31b4695a16:43093 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3705dec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f31b4695a16/172.17.0.2:0 2024-11-20T22:34:39,037 DEBUG [RS:1;9f31b4695a16:43815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e477ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f31b4695a16/172.17.0.2:0 2024-11-20T22:34:39,035 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9f31b4695a16,40769,1732142076492 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T22:34:39,042 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9f31b4695a16:0, corePoolSize=5, maxPoolSize=5 2024-11-20T22:34:39,042 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9f31b4695a16:0, corePoolSize=5, maxPoolSize=5 2024-11-20T22:34:39,042 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9f31b4695a16:0, corePoolSize=5, maxPoolSize=5 2024-11-20T22:34:39,042 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9f31b4695a16:0, corePoolSize=5, maxPoolSize=5 2024-11-20T22:34:39,042 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9f31b4695a16:0, corePoolSize=10, maxPoolSize=10 2024-11-20T22:34:39,043 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,043 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9f31b4695a16:0, corePoolSize=2, maxPoolSize=2 2024-11-20T22:34:39,043 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,050 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732142109050 2024-11-20T22:34:39,051 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9f31b4695a16:43815 2024-11-20T22:34:39,052 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T22:34:39,054 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T22:34:39,055 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T22:34:39,055 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T22:34:39,055 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T22:34:39,055 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9f31b4695a16:36989 2024-11-20T22:34:39,056 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T22:34:39,056 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T22:34:39,056 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T22:34:39,058 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9f31b4695a16:43093 2024-11-20T22:34:39,058 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T22:34:39,058 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T22:34:39,058 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T22:34:39,058 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T22:34:39,058 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T22:34:39,058 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T22:34:39,058 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(2659): reportForDuty to master=9f31b4695a16,40769,1732142076492 with port=43815, startcode=1732142077481 2024-11-20T22:34:39,058 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(2659): reportForDuty to master=9f31b4695a16,40769,1732142076492 with port=36989, startcode=1732142077539 2024-11-20T22:34:39,059 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T22:34:39,059 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T22:34:39,059 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T22:34:39,059 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(2659): reportForDuty to master=9f31b4695a16,40769,1732142076492 with port=43093, startcode=1732142077335 2024-11-20T22:34:39,059 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,062 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T22:34:39,063 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T22:34:39,064 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T22:34:39,065 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:39,066 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T22:34:39,066 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T22:34:39,066 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T22:34:39,068 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.large.0-1732142079067,5,FailOnTimeoutGroup] 2024-11-20T22:34:39,071 DEBUG [RS:1;9f31b4695a16:43815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T22:34:39,071 DEBUG [RS:2;9f31b4695a16:36989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T22:34:39,071 DEBUG [RS:0;9f31b4695a16:43093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T22:34:39,074 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.small.0-1732142079069,5,FailOnTimeoutGroup] 2024-11-20T22:34:39,074 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,074 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T22:34:39,076 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,076 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741831_1007 (size=1321) 2024-11-20T22:34:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741831_1007 (size=1321) 2024-11-20T22:34:39,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741831_1007 (size=1321) 2024-11-20T22:34:39,134 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T22:34:39,134 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 2024-11-20T22:34:39,150 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52643, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T22:34:39,150 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35589, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T22:34:39,150 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T22:34:39,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,162 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(517): Registering regionserver=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741832_1008 (size=32) 2024-11-20T22:34:39,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741832_1008 (size=32) 2024-11-20T22:34:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741832_1008 (size=32) 2024-11-20T22:34:39,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:39,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T22:34:39,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T22:34:39,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:39,177 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,177 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(517): Registering regionserver=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:39,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T22:34:39,181 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 2024-11-20T22:34:39,181 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-20T22:34:39,181 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T22:34:39,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T22:34:39,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:39,183 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 2024-11-20T22:34:39,184 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-20T22:34:39,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,184 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T22:34:39,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(517): Registering regionserver=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:39,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T22:34:39,187 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 2024-11-20T22:34:39,188 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-20T22:34:39,188 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T22:34:39,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T22:34:39,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:39,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:39,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T22:34:39,195 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T22:34:39,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:39,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:39,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T22:34:39,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:39,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:39,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T22:34:39,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T22:34:39,203 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T22:34:39,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T22:34:39,215 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:39,216 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70979382, jitterRate=0.05767521262168884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T22:34:39,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732142079171Initializing all the Stores at 1732142079173 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142079173Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142079173Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142079173Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142079174 (+1 ms)Cleaning up temporary data from old regions at 1732142079202 (+28 ms)Region opened successfully at 1732142079220 (+18 ms) 2024-11-20T22:34:39,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T22:34:39,220 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T22:34:39,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T22:34:39,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T22:34:39,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T22:34:39,222 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T22:34:39,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732142079220Disabling compacts and flushes for region at 1732142079220Disabling writes for close at 1732142079221 (+1 ms)Writing region close event to WAL at 1732142079221Closed at 1732142079222 (+1 ms) 2024-11-20T22:34:39,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T22:34:39,225 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T22:34:39,225 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T22:34:39,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T22:34:39,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T22:34:39,241 DEBUG [RS:2;9f31b4695a16:36989 {}] zookeeper.ZKUtil(111): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,241 DEBUG [RS:0;9f31b4695a16:43093 {}] zookeeper.ZKUtil(111): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,241 WARN [RS:0;9f31b4695a16:43093 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T22:34:39,241 WARN [RS:2;9f31b4695a16:36989 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T22:34:39,242 INFO [RS:0;9f31b4695a16:43093 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:39,242 INFO [RS:2;9f31b4695a16:36989 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:39,242 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,242 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9f31b4695a16,43093,1732142077335] 2024-11-20T22:34:39,244 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T22:34:39,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9f31b4695a16,43815,1732142077481] 2024-11-20T22:34:39,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9f31b4695a16,36989,1732142077539] 2024-11-20T22:34:39,244 DEBUG [RS:1;9f31b4695a16:43815 {}] zookeeper.ZKUtil(111): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,244 WARN [RS:1;9f31b4695a16:43815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T22:34:39,244 INFO [RS:1;9f31b4695a16:43815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:39,245 DEBUG [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,269 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T22:34:39,269 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T22:34:39,269 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T22:34:39,285 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T22:34:39,288 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T22:34:39,288 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T22:34:39,293 INFO [RS:0;9f31b4695a16:43093 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T22:34:39,293 INFO [RS:1;9f31b4695a16:43815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T22:34:39,293 INFO [RS:2;9f31b4695a16:36989 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T22:34:39,293 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,293 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,293 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,297 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T22:34:39,301 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T22:34:39,301 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T22:34:39,305 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T22:34:39,305 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T22:34:39,305 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T22:34:39,307 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,307 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,307 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,307 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,307 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,307 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,307 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,307 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,307 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0, corePoolSize=2, maxPoolSize=2 2024-11-20T22:34:39,307 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0, corePoolSize=2, maxPoolSize=2 2024-11-20T22:34:39,308 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,308 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0, corePoolSize=2, maxPoolSize=2 2024-11-20T22:34:39,309 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:1;9f31b4695a16:43815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,309 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,310 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9f31b4695a16:0, corePoolSize=1, maxPoolSize=1 2024-11-20T22:34:39,310 DEBUG [RS:2;9f31b4695a16:36989 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,310 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,310 DEBUG [RS:0;9f31b4695a16:43093 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9f31b4695a16:0, corePoolSize=3, maxPoolSize=3 2024-11-20T22:34:39,317 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,317 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,317 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,317 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,317 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,317 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,36989,1732142077539-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T22:34:39,318 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,43815,1732142077481-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T22:34:39,320 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,320 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,320 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,320 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,321 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,321 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,43093,1732142077335-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T22:34:39,343 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T22:34:39,343 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T22:34:39,343 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T22:34:39,346 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,43815,1732142077481-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,43093,1732142077335-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,36989,1732142077539-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,346 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.Replication(171): 9f31b4695a16,36989,1732142077539 started 2024-11-20T22:34:39,347 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.Replication(171): 9f31b4695a16,43093,1732142077335 started 2024-11-20T22:34:39,347 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.Replication(171): 9f31b4695a16,43815,1732142077481 started 2024-11-20T22:34:39,369 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,370 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1482): Serving as 9f31b4695a16,43093,1732142077335, RpcServer on 9f31b4695a16/172.17.0.2:43093, sessionid=0x1015a628ac10001 2024-11-20T22:34:39,371 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T22:34:39,371 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,371 DEBUG [RS:0;9f31b4695a16:43093 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,372 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1482): Serving as 9f31b4695a16,36989,1732142077539, RpcServer on 9f31b4695a16/172.17.0.2:36989, sessionid=0x1015a628ac10003 2024-11-20T22:34:39,372 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T22:34:39,372 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,43093,1732142077335' 2024-11-20T22:34:39,372 DEBUG [RS:2;9f31b4695a16:36989 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,372 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,36989,1732142077539' 2024-11-20T22:34:39,372 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T22:34:39,372 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T22:34:39,374 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T22:34:39,374 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T22:34:39,374 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:39,375 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1482): Serving as 9f31b4695a16,43815,1732142077481, RpcServer on 9f31b4695a16/172.17.0.2:43815, sessionid=0x1015a628ac10002 2024-11-20T22:34:39,375 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T22:34:39,375 DEBUG [RS:1;9f31b4695a16:43815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,375 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,43815,1732142077481' 2024-11-20T22:34:39,375 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T22:34:39,378 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T22:34:39,378 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T22:34:39,378 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T22:34:39,378 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T22:34:39,378 DEBUG [RS:2;9f31b4695a16:36989 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9f31b4695a16,36989,1732142077539 2024-11-20T22:34:39,378 DEBUG [RS:0;9f31b4695a16:43093 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,378 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,36989,1732142077539' 2024-11-20T22:34:39,378 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,43093,1732142077335' 2024-11-20T22:34:39,378 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T22:34:39,378 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T22:34:39,379 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T22:34:39,380 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T22:34:39,380 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T22:34:39,380 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T22:34:39,380 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T22:34:39,380 DEBUG [RS:1;9f31b4695a16:43815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9f31b4695a16,43815,1732142077481 2024-11-20T22:34:39,380 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f31b4695a16,43815,1732142077481' 2024-11-20T22:34:39,380 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T22:34:39,381 DEBUG [RS:0;9f31b4695a16:43093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T22:34:39,381 DEBUG [RS:2;9f31b4695a16:36989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T22:34:39,381 INFO [RS:2;9f31b4695a16:36989 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T22:34:39,381 INFO [RS:0;9f31b4695a16:43093 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T22:34:39,381 INFO [RS:0;9f31b4695a16:43093 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T22:34:39,381 INFO [RS:2;9f31b4695a16:36989 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T22:34:39,381 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T22:34:39,381 DEBUG [RS:1;9f31b4695a16:43815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T22:34:39,382 INFO [RS:1;9f31b4695a16:43815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T22:34:39,382 INFO [RS:1;9f31b4695a16:43815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T22:34:39,395 WARN [9f31b4695a16:40769 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T22:34:39,488 INFO [RS:0;9f31b4695a16:43093 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:39,488 INFO [RS:2;9f31b4695a16:36989 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:39,488 INFO [RS:1;9f31b4695a16:43815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:39,492 INFO [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C36989%2C1732142077539, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs, maxLogs=32 2024-11-20T22:34:39,492 INFO [RS:0;9f31b4695a16:43093 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C43093%2C1732142077335, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs, maxLogs=32 2024-11-20T22:34:39,496 INFO [RS:1;9f31b4695a16:43815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C43815%2C1732142077481, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43815,1732142077481, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs, maxLogs=32 2024-11-20T22:34:39,515 DEBUG [RS:1;9f31b4695a16:43815 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43815,1732142077481/9f31b4695a16%2C43815%2C1732142077481.1732142079500, exclude list is [], retry=0 2024-11-20T22:34:39,517 DEBUG [RS:2;9f31b4695a16:36989 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539/9f31b4695a16%2C36989%2C1732142077539.1732142079497, exclude list is [], retry=0 2024-11-20T22:34:39,522 DEBUG [RS:0;9f31b4695a16:43093 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335/9f31b4695a16%2C43093%2C1732142077335.1732142079497, exclude list is [], retry=0 2024-11-20T22:34:39,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:39,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:39,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:39,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:39,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:39,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:39,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:39,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:39,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:39,598 INFO [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539/9f31b4695a16%2C36989%2C1732142077539.1732142079497 2024-11-20T22:34:39,599 DEBUG [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:39,601 INFO [RS:1;9f31b4695a16:43815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43815,1732142077481/9f31b4695a16%2C43815%2C1732142077481.1732142079500 2024-11-20T22:34:39,602 DEBUG [RS:1;9f31b4695a16:43815 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:39,608 INFO [RS:0;9f31b4695a16:43093 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335/9f31b4695a16%2C43093%2C1732142077335.1732142079497 2024-11-20T22:34:39,609 DEBUG [RS:0;9f31b4695a16:43093 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:39,648 DEBUG [9f31b4695a16:40769 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T22:34:39,656 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(204): Hosts are {9f31b4695a16=0} racks are {/default-rack=0} 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T22:34:39,665 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T22:34:39,665 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T22:34:39,665 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T22:34:39,665 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T22:34:39,674 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:39,681 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f31b4695a16,43093,1732142077335, state=OPENING 2024-11-20T22:34:39,730 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T22:34:39,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:39,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:39,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:39,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:34:39,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:39,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:39,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:39,742 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:39,744 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T22:34:39,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9f31b4695a16,43093,1732142077335}] 2024-11-20T22:34:39,927 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T22:34:39,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T22:34:39,948 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T22:34:39,949 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:39,950 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T22:34:39,955 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C43093%2C1732142077335.meta, suffix=.meta, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs, maxLogs=32 2024-11-20T22:34:39,971 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, exclude list is [], retry=0 2024-11-20T22:34:39,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:39,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:39,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:40,023 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta 2024-11-20T22:34:40,024 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:40,024 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:40,026 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T22:34:40,030 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T22:34:40,036 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T22:34:40,042 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T22:34:40,043 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:40,043 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T22:34:40,043 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T22:34:40,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T22:34:40,055 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T22:34:40,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:40,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:40,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T22:34:40,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T22:34:40,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:40,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:40,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T22:34:40,063 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T22:34:40,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:40,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:40,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T22:34:40,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T22:34:40,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:40,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:40,067 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T22:34:40,069 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:40,072 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:40,075 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T22:34:40,075 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T22:34:40,076 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T22:34:40,085 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T22:34:40,087 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59841508, jitterRate=-0.10829204320907593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T22:34:40,087 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T22:34:40,089 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732142080044Writing region info on filesystem at 1732142080044Initializing all the Stores at 1732142080046 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142080047 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142080052 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142080052Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142080052Cleaning up temporary data from old regions at 1732142080075 (+23 ms)Running coprocessor post-open hooks at 1732142080087 (+12 ms)Region opened successfully at 1732142080089 (+2 ms) 2024-11-20T22:34:40,098 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732142079916 2024-11-20T22:34:40,112 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T22:34:40,112 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T22:34:40,115 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:40,118 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f31b4695a16,43093,1732142077335, state=OPEN 2024-11-20T22:34:40,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:40,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:40,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:40,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:40,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:40,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:40,173 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:40,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:40,174 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:40,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T22:34:40,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9f31b4695a16,43093,1732142077335 in 427 msec 2024-11-20T22:34:40,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T22:34:40,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 950 msec 2024-11-20T22:34:40,190 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T22:34:40,190 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T22:34:40,214 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:40,216 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1] 2024-11-20T22:34:40,243 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T22:34:40,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T22:34:40,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3590 sec 2024-11-20T22:34:40,315 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732142080315, completionTime=-1 2024-11-20T22:34:40,318 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T22:34:40,319 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T22:34:40,360 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T22:34:40,360 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732142140360 2024-11-20T22:34:40,360 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732142200360 2024-11-20T22:34:40,360 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-20T22:34:40,364 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T22:34:40,372 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,373 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,373 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,375 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9f31b4695a16:40769, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,375 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,376 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,384 DEBUG [master/9f31b4695a16:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T22:34:40,408 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.749sec 2024-11-20T22:34:40,410 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T22:34:40,412 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T22:34:40,414 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T22:34:40,414 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T22:34:40,415 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T22:34:40,416 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T22:34:40,416 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T22:34:40,425 DEBUG [master/9f31b4695a16:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T22:34:40,427 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T22:34:40,427 INFO [master/9f31b4695a16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f31b4695a16,40769,1732142076492-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:40,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd494aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T22:34:40,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f31b4695a16,40769,-1 for getting cluster id 2024-11-20T22:34:40,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T22:34:40,529 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6811c5ba-3839-4753-8916-d16cd6b50586' 2024-11-20T22:34:40,532 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T22:34:40,532 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6811c5ba-3839-4753-8916-d16cd6b50586" 2024-11-20T22:34:40,533 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c452a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T22:34:40,533 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f31b4695a16,40769,-1] 2024-11-20T22:34:40,537 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T22:34:40,539 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:34:40,541 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T22:34:40,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75555b5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T22:34:40,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:40,555 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1] 2024-11-20T22:34:40,555 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T22:34:40,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T22:34:40,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9f31b4695a16,40769,1732142076492 2024-11-20T22:34:40,586 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:46165/hbase 2024-11-20T22:34:40,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=1403 2024-11-20T22:34:40,636 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:40,641 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:40,642 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:40,648 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65734256, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-65734256, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:40,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65734256/hregion-65734256.1732142080649, exclude list is [], retry=0 2024-11-20T22:34:40,677 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:40,677 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:40,678 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:40,708 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65734256/hregion-65734256.1732142080649 2024-11-20T22:34:40,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:40,709 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fbb5e1fdb877056562820faaf4f73333, NAME => 'testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:34:40,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741838_1014 (size=64) 2024-11-20T22:34:40,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741838_1014 (size=64) 2024-11-20T22:34:40,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741838_1014 (size=64) 2024-11-20T22:34:40,723 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:40,725 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,729 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbb5e1fdb877056562820faaf4f73333 columnFamilyName a 2024-11-20T22:34:40,729 DEBUG [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:40,738 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] regionserver.HStore(327): Store=fbb5e1fdb877056562820faaf4f73333/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:40,738 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,740 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,742 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,742 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:40,758 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:40,760 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbb5e1fdb877056562820faaf4f73333; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69757520, jitterRate=0.03946805000305176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:34:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbb5e1fdb877056562820faaf4f73333: Writing region info on filesystem at 1732142080723Initializing all the Stores at 1732142080724 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142080724Cleaning up temporary data from old regions at 1732142080742 (+18 ms)Region opened successfully at 1732142080762 (+20 ms) 2024-11-20T22:34:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fbb5e1fdb877056562820faaf4f73333, disabling compactions & flushes 2024-11-20T22:34:40,762 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. 2024-11-20T22:34:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. 2024-11-20T22:34:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. after waiting 0 ms 2024-11-20T22:34:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. 2024-11-20T22:34:40,764 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. 2024-11-20T22:34:40,765 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fbb5e1fdb877056562820faaf4f73333: Waiting for close lock at 1732142080762Disabling compacts and flushes for region at 1732142080762Disabling writes for close at 1732142080762Writing region close event to WAL at 1732142080764 (+2 ms)Closed at 1732142080764 2024-11-20T22:34:40,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741837_1013 (size=175) 2024-11-20T22:34:40,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741837_1013 (size=175) 2024-11-20T22:34:40,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741837_1013 (size=175) 2024-11-20T22:34:40,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:40,778 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65734256:(num 1732142080649) 2024-11-20T22:34:40,781 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T22:34:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741839_1015 (size=469) 2024-11-20T22:34:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741839_1015 (size=469) 2024-11-20T22:34:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741839_1015 (size=469) 2024-11-20T22:34:40,830 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T22:34:40,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741840_1016 (size=322) 2024-11-20T22:34:40,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741840_1016 (size=322) 2024-11-20T22:34:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741840_1016 (size=322) 2024-11-20T22:34:40,905 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1, size=469 (469bytes) 2024-11-20T22:34:40,906 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T22:34:40,906 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T22:34:40,906 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1 2024-11-20T22:34:40,912 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1 after 4ms 2024-11-20T22:34:40,951 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:40,952 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1 took 48ms 2024-11-20T22:34:40,958 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1 so closing down 2024-11-20T22:34:40,958 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:40,964 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T22:34:40,999 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T22:34:41,001 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741841_1017 (size=469) 2024-11-20T22:34:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741841_1017 (size=469) 2024-11-20T22:34:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741841_1017 (size=469) 2024-11-20T22:34:41,018 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:41,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002 2024-11-20T22:34:41,026 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-20T22:34:41,027 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1, size=469 (469bytes) at 1732142080905Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1 so closing down at 1732142080958 (+53 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp at 1732142081000 (+42 ms)3 split writer threads finished at 1732142081001 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732142081018 (+17 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002 at 1732142081021 (+3 ms)Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732142081026 (+5 ms) 2024-11-20T22:34:41,049 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2, size=322 (322bytes) 2024-11-20T22:34:41,049 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2 2024-11-20T22:34:41,050 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2 after 1ms 2024-11-20T22:34:41,081 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:41,081 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2 took 32ms 2024-11-20T22:34:41,087 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2 so closing down 2024-11-20T22:34:41,087 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:41,090 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T22:34:41,125 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T22:34:41,125 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741842_1018 (size=322) 2024-11-20T22:34:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741842_1018 (size=322) 2024-11-20T22:34:41,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741842_1018 (size=322) 2024-11-20T22:34:41,160 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:41,204 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:41,208 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-20T22:34:41,210 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 126 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-20T22:34:41,210 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2, size=322 (322bytes) at 1732142081049Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2 so closing down at 1732142081087 (+38 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002-wal-2.temp at 1732142081125 (+38 ms)3 split writer threads finished at 1732142081125Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732142081160 (+35 ms)Processed 1 edits across 1 Regions in 126 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732142081210 (+50 ms) 2024-11-20T22:34:41,211 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:41,214 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:41,234 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal.1732142081215, exclude list is [], retry=0 2024-11-20T22:34:41,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:41,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:41,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:41,289 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal.1732142081215 2024-11-20T22:34:41,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:41,292 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fbb5e1fdb877056562820faaf4f73333, NAME => 'testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:41,292 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:41,292 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,293 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,296 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,298 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbb5e1fdb877056562820faaf4f73333 columnFamilyName a 2024-11-20T22:34:41,298 DEBUG [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:41,313 INFO [StoreOpener-fbb5e1fdb877056562820faaf4f73333-1 {}] regionserver.HStore(327): Store=fbb5e1fdb877056562820faaf4f73333/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:41,314 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,320 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,322 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002 2024-11-20T22:34:41,358 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:41,364 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002 2024-11-20T22:34:41,367 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbb5e1fdb877056562820faaf4f73333 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T22:34:41,421 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/.tmp/a/50638f7553d34ae5b5739b7f6f65e235 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732142080779/Put/seqid=0 2024-11-20T22:34:41,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741844_1020 (size=5170) 2024-11-20T22:34:41,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741844_1020 (size=5170) 2024-11-20T22:34:41,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741844_1020 (size=5170) 2024-11-20T22:34:41,434 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/.tmp/a/50638f7553d34ae5b5739b7f6f65e235 2024-11-20T22:34:41,485 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/.tmp/a/50638f7553d34ae5b5739b7f6f65e235 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/a/50638f7553d34ae5b5739b7f6f65e235 2024-11-20T22:34:41,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/a/50638f7553d34ae5b5739b7f6f65e235, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T22:34:41,508 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fbb5e1fdb877056562820faaf4f73333 in 138ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T22:34:41,510 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/0000000000000000002 2024-11-20T22:34:41,511 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,511 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,516 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbb5e1fdb877056562820faaf4f73333 2024-11-20T22:34:41,521 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/fbb5e1fdb877056562820faaf4f73333/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T22:34:41,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbb5e1fdb877056562820faaf4f73333; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69171888, jitterRate=0.030741453170776367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:34:41,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbb5e1fdb877056562820faaf4f73333: Writing region info on filesystem at 1732142081293Initializing all the Stores at 1732142081295 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142081296 (+1 ms)Obtaining lock to block concurrent updates at 1732142081367 (+71 ms)Preparing flush snapshotting stores in fbb5e1fdb877056562820faaf4f73333 at 1732142081367Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732142081371 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732142080638.fbb5e1fdb877056562820faaf4f73333. at 1732142081371Flushing fbb5e1fdb877056562820faaf4f73333/a: creating writer at 1732142081373 (+2 ms)Flushing fbb5e1fdb877056562820faaf4f73333/a: appending metadata at 1732142081408 (+35 ms)Flushing fbb5e1fdb877056562820faaf4f73333/a: closing flushed file at 1732142081410 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571a043a: reopening flushed file at 1732142081483 (+73 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for fbb5e1fdb877056562820faaf4f73333 in 138ms, sequenceid=2, compaction requested=false; wal=null at 1732142081508 (+25 ms)Cleaning up temporary data from old regions at 1732142081511 (+3 ms)Region opened successfully at 1732142081525 (+14 ms) 2024-11-20T22:34:41,554 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:44526 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:44702 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:43108 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:35800 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:35816 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46165/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:43090 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=701 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 431) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1389 (was 1403) 2024-11-20T22:34:41,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=701, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=1388 2024-11-20T22:34:41,588 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:41,591 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:41,592 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:41,597 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05594175, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-05594175, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:41,614 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05594175/hregion-05594175.1732142081597, exclude list is [], retry=0 2024-11-20T22:34:41,618 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:41,618 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:41,628 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:41,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05594175/hregion-05594175.1732142081597 2024-11-20T22:34:41,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:41,674 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dc6eca6817df586fc75a05e7d370b65b, NAME => 'testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:34:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741846_1022 (size=64) 2024-11-20T22:34:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741846_1022 (size=64) 2024-11-20T22:34:41,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741846_1022 (size=64) 2024-11-20T22:34:41,690 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:41,692 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,695 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6eca6817df586fc75a05e7d370b65b columnFamilyName a 2024-11-20T22:34:41,695 DEBUG [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:41,696 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] regionserver.HStore(327): Store=dc6eca6817df586fc75a05e7d370b65b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:41,696 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,698 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,699 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,699 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,702 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:41,706 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:41,707 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dc6eca6817df586fc75a05e7d370b65b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74594775, jitterRate=0.1115487664937973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:34:41,707 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dc6eca6817df586fc75a05e7d370b65b: Writing region info on filesystem at 1732142081690Initializing all the Stores at 1732142081691 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142081691Cleaning up temporary data from old regions at 1732142081699 (+8 ms)Region opened successfully at 1732142081707 (+8 ms) 2024-11-20T22:34:41,707 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dc6eca6817df586fc75a05e7d370b65b, disabling compactions & flushes 2024-11-20T22:34:41,707 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. 2024-11-20T22:34:41,707 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. 2024-11-20T22:34:41,707 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. after waiting 0 ms 2024-11-20T22:34:41,707 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. 2024-11-20T22:34:41,708 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. 2024-11-20T22:34:41,708 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dc6eca6817df586fc75a05e7d370b65b: Waiting for close lock at 1732142081707Disabling compacts and flushes for region at 1732142081707Disabling writes for close at 1732142081707Writing region close event to WAL at 1732142081708 (+1 ms)Closed at 1732142081708 2024-11-20T22:34:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741845_1021 (size=175) 2024-11-20T22:34:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741845_1021 (size=175) 2024-11-20T22:34:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741845_1021 (size=175) 2024-11-20T22:34:41,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:41,714 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05594175:(num 1732142081597) 2024-11-20T22:34:41,715 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T22:34:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741847_1023 (size=469) 2024-11-20T22:34:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741847_1023 (size=469) 2024-11-20T22:34:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741847_1023 (size=469) 2024-11-20T22:34:41,757 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T22:34:41,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741848_1024 (size=322) 2024-11-20T22:34:41,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741848_1024 (size=322) 2024-11-20T22:34:41,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741848_1024 (size=322) 2024-11-20T22:34:41,814 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2, size=322 (322bytes) 2024-11-20T22:34:41,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2 2024-11-20T22:34:41,816 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2 after 1ms 2024-11-20T22:34:41,858 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:41,859 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2 took 45ms 2024-11-20T22:34:41,862 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2 so closing down 2024-11-20T22:34:41,862 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:41,865 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T22:34:41,893 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T22:34:41,893 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:41,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741849_1025 (size=322) 2024-11-20T22:34:41,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741849_1025 (size=322) 2024-11-20T22:34:41,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741849_1025 (size=322) 2024-11-20T22:34:41,904 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:41,906 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 2024-11-20T22:34:41,906 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-20T22:34:41,906 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2, size=322 (322bytes) at 1732142081814Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2 so closing down at 1732142081862 (+48 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp at 1732142081893 (+31 ms)3 split writer threads finished at 1732142081893Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732142081904 (+11 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 at 1732142081906 (+2 ms)Processed 1 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732142081906 2024-11-20T22:34:41,921 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1, size=469 (469bytes) 2024-11-20T22:34:41,921 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1 2024-11-20T22:34:41,922 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1 after 1ms 2024-11-20T22:34:41,957 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:41,957 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1 took 37ms 2024-11-20T22:34:41,961 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1 so closing down 2024-11-20T22:34:41,961 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:41,964 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T22:34:41,995 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T22:34:41,995 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:42,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741850_1026 (size=469) 2024-11-20T22:34:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741850_1026 (size=469) 2024-11-20T22:34:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741850_1026 (size=469) 2024-11-20T22:34:42,006 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:42,038 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:42,041 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002, length=322 2024-11-20T22:34:42,044 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 2024-11-20T22:34:42,044 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-20T22:34:42,044 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1, size=469 (469bytes) at 1732142081921Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1 so closing down at 1732142081961 (+40 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp at 1732142081995 (+34 ms)3 split writer threads finished at 1732142081996 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732142082006 (+10 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 at 1732142082044 (+38 ms)Processed 2 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732142082044 2024-11-20T22:34:42,045 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:42,047 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:42,060 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal.1732142082047, exclude list is [], retry=0 2024-11-20T22:34:42,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:42,065 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:42,065 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:42,099 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal.1732142082047 2024-11-20T22:34:42,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:42,100 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dc6eca6817df586fc75a05e7d370b65b, NAME => 'testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:42,101 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:42,101 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,101 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,104 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,107 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6eca6817df586fc75a05e7d370b65b columnFamilyName a 2024-11-20T22:34:42,107 DEBUG [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:42,108 INFO [StoreOpener-dc6eca6817df586fc75a05e7d370b65b-1 {}] regionserver.HStore(327): Store=dc6eca6817df586fc75a05e7d370b65b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:42,108 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,115 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 2024-11-20T22:34:42,147 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:42,149 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 2024-11-20T22:34:42,150 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dc6eca6817df586fc75a05e7d370b65b 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T22:34:42,167 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/.tmp/a/68804515680a488da644e253b6c63b7c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732142081714/Put/seqid=0 2024-11-20T22:34:42,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741852_1028 (size=5170) 2024-11-20T22:34:42,183 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/.tmp/a/68804515680a488da644e253b6c63b7c 2024-11-20T22:34:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741852_1028 (size=5170) 2024-11-20T22:34:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741852_1028 (size=5170) 2024-11-20T22:34:42,198 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/.tmp/a/68804515680a488da644e253b6c63b7c as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/a/68804515680a488da644e253b6c63b7c 2024-11-20T22:34:42,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/a/68804515680a488da644e253b6c63b7c, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T22:34:42,210 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dc6eca6817df586fc75a05e7d370b65b in 60ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T22:34:42,211 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/0000000000000000002 2024-11-20T22:34:42,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,215 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dc6eca6817df586fc75a05e7d370b65b 2024-11-20T22:34:42,218 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/dc6eca6817df586fc75a05e7d370b65b/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T22:34:42,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dc6eca6817df586fc75a05e7d370b65b; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68510799, jitterRate=0.020890459418296814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:34:42,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dc6eca6817df586fc75a05e7d370b65b: Writing region info on filesystem at 1732142082101Initializing all the Stores at 1732142082103 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142082104 (+1 ms)Obtaining lock to block concurrent updates at 1732142082150 (+46 ms)Preparing flush snapshotting stores in dc6eca6817df586fc75a05e7d370b65b at 1732142082150Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732142082150Flushing stores of testReplayEditsWrittenIntoWAL,,1732142081589.dc6eca6817df586fc75a05e7d370b65b. at 1732142082150Flushing dc6eca6817df586fc75a05e7d370b65b/a: creating writer at 1732142082150Flushing dc6eca6817df586fc75a05e7d370b65b/a: appending metadata at 1732142082166 (+16 ms)Flushing dc6eca6817df586fc75a05e7d370b65b/a: closing flushed file at 1732142082166Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a8a22d2: reopening flushed file at 1732142082196 (+30 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dc6eca6817df586fc75a05e7d370b65b in 60ms, sequenceid=2, compaction requested=false; wal=null at 1732142082210 (+14 ms)Cleaning up temporary data from old regions at 1732142082211 (+1 ms)Region opened successfully at 1732142082220 (+9 ms) 2024-11-20T22:34:42,248 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48364 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:35816 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:59680 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:43090 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:44526 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:49146 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 701) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=1371 (was 1388) 2024-11-20T22:34:42,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=1370 2024-11-20T22:34:42,285 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:42,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:42,289 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:42,295 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37837576, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-37837576, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:42,311 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37837576/hregion-37837576.1732142082298, exclude list is [], retry=0 2024-11-20T22:34:42,316 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:42,317 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:42,317 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:42,358 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37837576/hregion-37837576.1732142082298 2024-11-20T22:34:42,358 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:34:42,359 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 45221c203677012da91485aac4497a68, NAME => 'testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:34:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741854_1030 (size=64) 2024-11-20T22:34:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741854_1030 (size=64) 2024-11-20T22:34:42,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741854_1030 (size=64) 2024-11-20T22:34:42,375 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:42,377 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,379 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName a 2024-11-20T22:34:42,379 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:42,380 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:42,380 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,382 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName b 2024-11-20T22:34:42,382 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:42,383 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:42,383 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,385 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName c 2024-11-20T22:34:42,385 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:42,385 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:42,386 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68 2024-11-20T22:34:42,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68 2024-11-20T22:34:42,390 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,390 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,391 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:42,394 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45221c203677012da91485aac4497a68 2024-11-20T22:34:42,397 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:42,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45221c203677012da91485aac4497a68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61760123, jitterRate=-0.0797024518251419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:42,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45221c203677012da91485aac4497a68: Writing region info on filesystem at 1732142082375Initializing all the Stores at 1732142082376 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142082376Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142082377 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142082377Cleaning up temporary data from old regions at 1732142082390 (+13 ms)Region opened successfully at 1732142082398 (+8 ms) 2024-11-20T22:34:42,398 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45221c203677012da91485aac4497a68, disabling compactions & flushes 2024-11-20T22:34:42,398 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:42,398 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:42,398 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. after waiting 0 ms 2024-11-20T22:34:42,398 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:42,399 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:42,399 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45221c203677012da91485aac4497a68: Waiting for close lock at 1732142082398Disabling compacts and flushes for region at 1732142082398Disabling writes for close at 1732142082398Writing region close event to WAL at 1732142082399 (+1 ms)Closed at 1732142082399 2024-11-20T22:34:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741853_1029 (size=175) 2024-11-20T22:34:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741853_1029 (size=175) 2024-11-20T22:34:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741853_1029 (size=175) 2024-11-20T22:34:42,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:42,406 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37837576:(num 1732142082298) 2024-11-20T22:34:42,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:42,409 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:42,420 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, exclude list is [], retry=0 2024-11-20T22:34:42,424 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:42,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:42,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:42,465 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 2024-11-20T22:34:42,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:34:42,729 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, size=0 (0bytes) 2024-11-20T22:34:42,730 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 might be still open, length is 0 2024-11-20T22:34:42,730 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 2024-11-20T22:34:42,732 WARN [IPC Server handler 3 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-20T22:34:42,733 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 after 3ms 2024-11-20T22:34:44,531 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:49194 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37697:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49194 dst: /127.0.0.1:37697 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37697 remote=/127.0.0.1:49194]. Total timeout mills is 60000, 58151 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:44,532 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:59718 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59718 dst: /127.0.0.1:36643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:44,532 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48418 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48418 dst: /127.0.0.1:45551 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:44,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741855_1032 (size=497775) 2024-11-20T22:34:44,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741855_1032 (size=497775) 2024-11-20T22:34:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741855_1032 (size=497775) 2024-11-20T22:34:45,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T22:34:45,614 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T22:34:46,734 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 after 4004ms 2024-11-20T22:34:46,764 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:46,766 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 took 4037ms 2024-11-20T22:34:46,771 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732142082409.temp 2024-11-20T22:34:46,805 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp 2024-11-20T22:34:46,850 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T22:34:46,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T22:34:46,852 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T22:34:46,853 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T22:34:46,853 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T22:34:46,853 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T22:34:46,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-20T22:34:46,854 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-20T22:34:46,876 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409; continuing. 2024-11-20T22:34:46,877 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 so closing down 2024-11-20T22:34:46,877 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:46,879 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:46,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741856_1033 (size=497783) 2024-11-20T22:34:46,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741856_1033 (size=497783) 2024-11-20T22:34:46,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741856_1033 (size=497783) 2024-11-20T22:34:46,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp (wrote 3002 edits, skipped 0 edits in 47 ms) 2024-11-20T22:34:46,888 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 2024-11-20T22:34:46,888 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T22:34:46,888 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, size=0 (0bytes) at 1732142082730Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp at 1732142086806 (+4076 ms)Split 1024 edits, skipped 0 edits. at 1732142086833 (+27 ms)Split 2048 edits, skipped 0 edits. at 1732142086858 (+25 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 so closing down at 1732142086877 (+19 ms)3 split writer threads finished at 1732142086879 (+2 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp (wrote 3002 edits, skipped 0 edits in 47 ms) at 1732142086885 (+6 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000000001-wal.1732142082409.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 at 1732142086888 (+3 ms)Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409, size=0, length=0, corrupted=false, cancelled=false at 1732142086888 2024-11-20T22:34:46,891 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142082409 2024-11-20T22:34:46,893 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 2024-11-20T22:34:46,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:46,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:46,909 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142086896, exclude list is [], retry=0 2024-11-20T22:34:46,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:46,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:46,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:46,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142086896 2024-11-20T22:34:46,941 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:46,941 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:46,943 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:46,945 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName a 2024-11-20T22:34:46,945 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:46,946 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:46,946 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:46,947 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName b 2024-11-20T22:34:46,947 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:46,948 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:46,948 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45221c203677012da91485aac4497a68 2024-11-20T22:34:46,949 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45221c203677012da91485aac4497a68 columnFamilyName c 2024-11-20T22:34:46,949 DEBUG [StoreOpener-45221c203677012da91485aac4497a68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:46,950 INFO [StoreOpener-45221c203677012da91485aac4497a68-1 {}] regionserver.HStore(327): Store=45221c203677012da91485aac4497a68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:46,950 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45221c203677012da91485aac4497a68 2024-11-20T22:34:46,951 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68 2024-11-20T22:34:46,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68 2024-11-20T22:34:46,954 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 2024-11-20T22:34:46,981 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:47,033 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T22:34:47,437 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45221c203677012da91485aac4497a68 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T22:34:47,491 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/a462983a7f444ffda885da9e846f10c8 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732142082477/Put/seqid=0 2024-11-20T22:34:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741858_1035 (size=50463) 2024-11-20T22:34:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741858_1035 (size=50463) 2024-11-20T22:34:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741858_1035 (size=50463) 2024-11-20T22:34:47,502 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/a462983a7f444ffda885da9e846f10c8 2024-11-20T22:34:47,510 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/a462983a7f444ffda885da9e846f10c8 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/a/a462983a7f444ffda885da9e846f10c8 2024-11-20T22:34:47,521 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/a/a462983a7f444ffda885da9e846f10c8, entries=754, sequenceid=754, filesize=49.3 K 2024-11-20T22:34:47,522 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 45221c203677012da91485aac4497a68 in 85ms, sequenceid=754, compaction requested=false; wal=null 2024-11-20T22:34:47,546 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T22:34:47,547 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45221c203677012da91485aac4497a68 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T22:34:47,557 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/c34a1d323a5746448903fd198b54e8dc is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732142082536/Put/seqid=0 2024-11-20T22:34:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741859_1036 (size=20072) 2024-11-20T22:34:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741859_1036 (size=20072) 2024-11-20T22:34:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741859_1036 (size=20072) 2024-11-20T22:34:47,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/c34a1d323a5746448903fd198b54e8dc 2024-11-20T22:34:47,592 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/221fa36aee6247f8be08fd3599ea836d is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732142082566/Put/seqid=0 2024-11-20T22:34:47,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741860_1037 (size=35835) 2024-11-20T22:34:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741860_1037 (size=35835) 2024-11-20T22:34:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741860_1037 (size=35835) 2024-11-20T22:34:47,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/221fa36aee6247f8be08fd3599ea836d 2024-11-20T22:34:47,608 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/a/c34a1d323a5746448903fd198b54e8dc as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/a/c34a1d323a5746448903fd198b54e8dc 2024-11-20T22:34:47,621 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/a/c34a1d323a5746448903fd198b54e8dc, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-20T22:34:47,623 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/221fa36aee6247f8be08fd3599ea836d as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/b/221fa36aee6247f8be08fd3599ea836d 2024-11-20T22:34:47,635 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/b/221fa36aee6247f8be08fd3599ea836d, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-20T22:34:47,635 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 45221c203677012da91485aac4497a68 in 89ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-20T22:34:47,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T22:34:47,658 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45221c203677012da91485aac4497a68 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T22:34:47,669 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/03128d2a2bb04547b6391d10d7bd7498 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732142082587/Put/seqid=0 2024-11-20T22:34:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741861_1038 (size=35082) 2024-11-20T22:34:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741861_1038 (size=35082) 2024-11-20T22:34:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741861_1038 (size=35082) 2024-11-20T22:34:47,686 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/03128d2a2bb04547b6391d10d7bd7498 2024-11-20T22:34:47,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/13bb92661761473bbdb3e5b21aebd052 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732142082623/Put/seqid=0 2024-11-20T22:34:47,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741862_1039 (size=20825) 2024-11-20T22:34:47,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741862_1039 (size=20825) 2024-11-20T22:34:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741862_1039 (size=20825) 2024-11-20T22:34:47,725 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/13bb92661761473bbdb3e5b21aebd052 2024-11-20T22:34:47,733 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/b/03128d2a2bb04547b6391d10d7bd7498 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/b/03128d2a2bb04547b6391d10d7bd7498 2024-11-20T22:34:47,742 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/b/03128d2a2bb04547b6391d10d7bd7498, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-20T22:34:47,744 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/13bb92661761473bbdb3e5b21aebd052 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/c/13bb92661761473bbdb3e5b21aebd052 2024-11-20T22:34:47,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/c/13bb92661761473bbdb3e5b21aebd052, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-20T22:34:47,754 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 45221c203677012da91485aac4497a68 in 95ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-20T22:34:47,770 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732142082679/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:47,774 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 2024-11-20T22:34:47,775 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T22:34:47,775 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45221c203677012da91485aac4497a68 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-20T22:34:47,785 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/ea3b4c5aae764c61b5e69f7e1b96782f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732142082629/Put/seqid=0 2024-11-20T22:34:47,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741863_1040 (size=50301) 2024-11-20T22:34:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741863_1040 (size=50301) 2024-11-20T22:34:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741863_1040 (size=50301) 2024-11-20T22:34:47,795 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/ea3b4c5aae764c61b5e69f7e1b96782f 2024-11-20T22:34:47,803 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ea3b4c5aae764c61b5e69f7e1b96782f 2024-11-20T22:34:47,805 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/.tmp/c/ea3b4c5aae764c61b5e69f7e1b96782f as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/c/ea3b4c5aae764c61b5e69f7e1b96782f 2024-11-20T22:34:47,815 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ea3b4c5aae764c61b5e69f7e1b96782f 2024-11-20T22:34:47,815 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/c/ea3b4c5aae764c61b5e69f7e1b96782f, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-20T22:34:47,815 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 45221c203677012da91485aac4497a68 in 40ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-20T22:34:47,817 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/0000000000000003002 2024-11-20T22:34:47,819 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45221c203677012da91485aac4497a68 2024-11-20T22:34:47,819 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45221c203677012da91485aac4497a68 2024-11-20T22:34:47,820 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T22:34:47,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45221c203677012da91485aac4497a68 2024-11-20T22:34:47,833 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/45221c203677012da91485aac4497a68/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-20T22:34:47,835 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45221c203677012da91485aac4497a68; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62025089, jitterRate=-0.07575415074825287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T22:34:47,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45221c203677012da91485aac4497a68: Writing region info on filesystem at 1732142086941Initializing all the Stores at 1732142086943 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142086943Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142086943Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142086943Cleaning up temporary data from old regions at 1732142087819 (+876 ms)Region opened successfully at 1732142087835 (+16 ms) 2024-11-20T22:34:47,926 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45221c203677012da91485aac4497a68, disabling compactions & flushes 2024-11-20T22:34:47,926 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:47,926 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:47,926 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. after waiting 0 ms 2024-11-20T22:34:47,926 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:47,941 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732142082286.45221c203677012da91485aac4497a68. 2024-11-20T22:34:47,941 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45221c203677012da91485aac4497a68: Waiting for close lock at 1732142087925Disabling compacts and flushes for region at 1732142087925Disabling writes for close at 1732142087926 (+1 ms)Writing region close event to WAL at 1732142087941 (+15 ms)Closed at 1732142087941 2024-11-20T22:34:47,951 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142086896 not finished, retry = 0 2024-11-20T22:34:47,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741857_1034 (size=175) 2024-11-20T22:34:47,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741857_1034 (size=175) 2024-11-20T22:34:47,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741857_1034 (size=175) 2024-11-20T22:34:48,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:48,055 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732142086896) 2024-11-20T22:34:48,070 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=406 (was 390) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1011129840_22 at /127.0.0.1:48476 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43593 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46117 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1011129840_22 at /127.0.0.1:59780 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43593 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46165 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1011129840_22 at /127.0.0.1:48524 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46117 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1011129840_22 at /127.0.0.1:49222 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=1256 (was 1370) 2024-11-20T22:34:48,083 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=406, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=1255 2024-11-20T22:34:48,101 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:48,104 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:48,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:48,108 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25595634, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-25595634, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:48,121 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25595634/hregion-25595634.1732142088109, exclude list is [], retry=0 2024-11-20T22:34:48,124 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:48,125 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:48,125 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:48,154 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25595634/hregion-25595634.1732142088109 2024-11-20T22:34:48,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:48,155 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f3153ae3eb4a131f285356ff640ae010, NAME => 'test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:34:48,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741865_1042 (size=43) 2024-11-20T22:34:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741865_1042 (size=43) 2024-11-20T22:34:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741865_1042 (size=43) 2024-11-20T22:34:48,166 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:48,167 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,169 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName a 2024-11-20T22:34:48,169 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:48,169 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:48,169 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,171 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName b 2024-11-20T22:34:48,171 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:48,172 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:48,172 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,174 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName c 2024-11-20T22:34:48,174 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:48,174 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:48,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,177 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,177 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,178 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:48,179 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:48,182 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:48,183 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3153ae3eb4a131f285356ff640ae010; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64382857, jitterRate=-0.04062066972255707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3153ae3eb4a131f285356ff640ae010: Writing region info on filesystem at 1732142088166Initializing all the Stores at 1732142088167 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142088167Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142088167Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142088167Cleaning up temporary data from old regions at 1732142088177 (+10 ms)Region opened successfully at 1732142088184 (+7 ms) 2024-11-20T22:34:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3153ae3eb4a131f285356ff640ae010, disabling compactions & flushes 2024-11-20T22:34:48,184 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. after waiting 0 ms 2024-11-20T22:34:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:48,184 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:48,185 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3153ae3eb4a131f285356ff640ae010: Waiting for close lock at 1732142088184Disabling compacts and flushes for region at 1732142088184Disabling writes for close at 1732142088184Writing region close event to WAL at 1732142088184Closed at 1732142088184 2024-11-20T22:34:48,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741864_1041 (size=175) 2024-11-20T22:34:48,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741864_1041 (size=175) 2024-11-20T22:34:48,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741864_1041 (size=175) 2024-11-20T22:34:48,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:48,192 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25595634:(num 1732142088109) 2024-11-20T22:34:48,192 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:48,194 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:48,207 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, exclude list is [], retry=0 2024-11-20T22:34:48,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:48,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:48,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:48,241 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 2024-11-20T22:34:48,242 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:34:48,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741866_1043 (size=371388) 2024-11-20T22:34:48,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741866_1043 (size=371388) 2024-11-20T22:34:48,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741866_1043 (size=371388) 2024-11-20T22:34:48,440 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, size=362.7 K (371388bytes) 2024-11-20T22:34:48,440 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 2024-11-20T22:34:48,441 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 after 1ms 2024-11-20T22:34:48,473 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:48,475 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 took 35ms 2024-11-20T22:34:48,481 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732142088194.temp 2024-11-20T22:34:48,518 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 so closing down 2024-11-20T22:34:48,518 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:48,525 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp 2024-11-20T22:34:48,542 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741867_1044 (size=371388) 2024-11-20T22:34:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741867_1044 (size=371388) 2024-11-20T22:34:48,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741867_1044 (size=371388) 2024-11-20T22:34:48,553 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-20T22:34:48,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp to hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 2024-11-20T22:34:48,556 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-20T22:34:48,556 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, journal: Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, size=362.7 K (371388bytes) at 1732142088440Split 1024 edits, skipped 0 edits. at 1732142088493 (+53 ms)Split 2048 edits, skipped 0 edits. at 1732142088505 (+12 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 so closing down at 1732142088518 (+13 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp at 1732142088525 (+7 ms)3 split writer threads finished at 1732142088542 (+17 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732142088553 (+11 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000000001-wal.1732142088194.temp to hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 at 1732142088555 (+2 ms)Processed 3000 edits across 1 Regions in 80 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732142088556 (+1 ms) 2024-11-20T22:34:48,559 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088194 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142088194 2024-11-20T22:34:48,560 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 2024-11-20T22:34:48,560 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:48,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:48,583 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, exclude list is [], retry=0 2024-11-20T22:34:48,586 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:48,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:48,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:48,620 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 2024-11-20T22:34:48,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:48,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741868_1045 (size=371515) 2024-11-20T22:34:48,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741868_1045 (size=371515) 2024-11-20T22:34:48,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741868_1045 (size=371515) 2024-11-20T22:34:48,807 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, size=362.8 K (371515bytes) 2024-11-20T22:34:48,807 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 2024-11-20T22:34:48,808 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 after 1ms 2024-11-20T22:34:48,875 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:48,877 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 took 70ms 2024-11-20T22:34:48,882 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732142088566.temp 2024-11-20T22:34:48,902 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 so closing down 2024-11-20T22:34:48,902 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:48,921 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp 2024-11-20T22:34:48,935 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:48,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741869_1046 (size=371515) 2024-11-20T22:34:48,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741869_1046 (size=371515) 2024-11-20T22:34:48,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741869_1046 (size=371515) 2024-11-20T22:34:48,947 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-20T22:34:48,949 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp to hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 2024-11-20T22:34:48,950 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-20T22:34:48,950 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, journal: Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, size=362.8 K (371515bytes) at 1732142088807Split 1024 edits, skipped 0 edits. at 1732142088887 (+80 ms)Split 2048 edits, skipped 0 edits. at 1732142088894 (+7 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 so closing down at 1732142088902 (+8 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp at 1732142088921 (+19 ms)3 split writer threads finished at 1732142088935 (+14 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1732142088947 (+12 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003001-wal.1732142088566.temp to hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 at 1732142088950 (+3 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732142088950 2024-11-20T22:34:48,953 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088566 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142088566 2024-11-20T22:34:48,954 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 2024-11-20T22:34:48,954 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:48,957 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732142088101, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:48,977 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088957, exclude list is [], retry=0 2024-11-20T22:34:48,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:48,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:48,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:49,032 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732142088101/wal.1732142088957 2024-11-20T22:34:49,032 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:34:49,032 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3153ae3eb4a131f285356ff640ae010, NAME => 'test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:49,033 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:49,033 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,033 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,041 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,043 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName a 2024-11-20T22:34:49,043 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:49,044 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:49,045 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,046 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName b 2024-11-20T22:34:49,046 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:49,049 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:49,049 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,051 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3153ae3eb4a131f285356ff640ae010 columnFamilyName c 2024-11-20T22:34:49,051 DEBUG [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:49,053 INFO [StoreOpener-f3153ae3eb4a131f285356ff640ae010-1 {}] regionserver.HStore(327): Store=f3153ae3eb4a131f285356ff640ae010/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:49,057 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,059 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,068 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 2024-11-20T22:34:49,106 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:49,149 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 2024-11-20T22:34:49,151 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 2024-11-20T22:34:49,192 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:49,231 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 2024-11-20T22:34:49,231 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3153ae3eb4a131f285356ff640ae010 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-20T22:34:49,266 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/a/64f002b390104c7ca3da9f855ba31aa3 is 41, key is test2727/a:100/1732142088625/Put/seqid=0 2024-11-20T22:34:49,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741871_1048 (size=84227) 2024-11-20T22:34:49,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741871_1048 (size=84227) 2024-11-20T22:34:49,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741871_1048 (size=84227) 2024-11-20T22:34:49,317 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/a/64f002b390104c7ca3da9f855ba31aa3 2024-11-20T22:34:49,363 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/b/9e4ae42093db4333b50564ee13da744c is 41, key is test2727/b:100/1732142088681/Put/seqid=0 2024-11-20T22:34:49,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741872_1049 (size=84609) 2024-11-20T22:34:49,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741872_1049 (size=84609) 2024-11-20T22:34:49,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741872_1049 (size=84609) 2024-11-20T22:34:49,400 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/b/9e4ae42093db4333b50564ee13da744c 2024-11-20T22:34:49,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/c/d7fb3926756549d2914ffd23df820a8a is 41, key is test2727/c:100/1732142088731/Put/seqid=0 2024-11-20T22:34:49,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741873_1050 (size=84609) 2024-11-20T22:34:49,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741873_1050 (size=84609) 2024-11-20T22:34:49,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741873_1050 (size=84609) 2024-11-20T22:34:49,480 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/c/d7fb3926756549d2914ffd23df820a8a 2024-11-20T22:34:49,497 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/a/64f002b390104c7ca3da9f855ba31aa3 as hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/a/64f002b390104c7ca3da9f855ba31aa3 2024-11-20T22:34:49,509 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/a/64f002b390104c7ca3da9f855ba31aa3, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-20T22:34:49,519 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/b/9e4ae42093db4333b50564ee13da744c as hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/b/9e4ae42093db4333b50564ee13da744c 2024-11-20T22:34:49,539 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/b/9e4ae42093db4333b50564ee13da744c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T22:34:49,545 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/.tmp/c/d7fb3926756549d2914ffd23df820a8a as hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/c/d7fb3926756549d2914ffd23df820a8a 2024-11-20T22:34:49,566 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/c/d7fb3926756549d2914ffd23df820a8a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T22:34:49,566 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f3153ae3eb4a131f285356ff640ae010 in 335ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-20T22:34:49,568 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000003000 2024-11-20T22:34:49,571 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/0000000000000006000 2024-11-20T22:34:49,572 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,572 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,574 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:49,576 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3153ae3eb4a131f285356ff640ae010 2024-11-20T22:34:49,584 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/test2727/f3153ae3eb4a131f285356ff640ae010/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-20T22:34:49,587 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3153ae3eb4a131f285356ff640ae010; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66173188, jitterRate=-0.0139426589012146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:49,588 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3153ae3eb4a131f285356ff640ae010: Writing region info on filesystem at 1732142089033Initializing all the Stores at 1732142089041 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142089041Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142089041Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142089041Obtaining lock to block concurrent updates at 1732142089231 (+190 ms)Preparing flush snapshotting stores in f3153ae3eb4a131f285356ff640ae010 at 1732142089231Finished memstore snapshotting test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732142089232 (+1 ms)Flushing stores of test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. at 1732142089232Flushing f3153ae3eb4a131f285356ff640ae010/a: creating writer at 1732142089232Flushing f3153ae3eb4a131f285356ff640ae010/a: appending metadata at 1732142089265 (+33 ms)Flushing f3153ae3eb4a131f285356ff640ae010/a: closing flushed file at 1732142089265Flushing f3153ae3eb4a131f285356ff640ae010/b: creating writer at 1732142089328 (+63 ms)Flushing f3153ae3eb4a131f285356ff640ae010/b: appending metadata at 1732142089359 (+31 ms)Flushing f3153ae3eb4a131f285356ff640ae010/b: closing flushed file at 1732142089359Flushing f3153ae3eb4a131f285356ff640ae010/c: creating writer at 1732142089409 (+50 ms)Flushing f3153ae3eb4a131f285356ff640ae010/c: appending metadata at 1732142089438 (+29 ms)Flushing f3153ae3eb4a131f285356ff640ae010/c: closing flushed file at 1732142089438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ab71a0: reopening flushed file at 1732142089495 (+57 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b08bae: reopening flushed file at 1732142089510 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@90d38ab: reopening flushed file at 1732142089539 (+29 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f3153ae3eb4a131f285356ff640ae010 in 335ms, sequenceid=6000, compaction requested=false; wal=null at 1732142089566 (+27 ms)Cleaning up temporary data from old regions at 1732142089572 (+6 ms)Region opened successfully at 1732142089588 (+16 ms) 2024-11-20T22:34:49,590 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-20T22:34:49,590 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3153ae3eb4a131f285356ff640ae010, disabling compactions & flushes 2024-11-20T22:34:49,590 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:49,590 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:49,590 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. after waiting 0 ms 2024-11-20T22:34:49,590 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:49,593 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732142088102.f3153ae3eb4a131f285356ff640ae010. 2024-11-20T22:34:49,593 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3153ae3eb4a131f285356ff640ae010: Waiting for close lock at 1732142089590Disabling compacts and flushes for region at 1732142089590Disabling writes for close at 1732142089590Writing region close event to WAL at 1732142089593 (+3 ms)Closed at 1732142089593 2024-11-20T22:34:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741870_1047 (size=175) 2024-11-20T22:34:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741870_1047 (size=175) 2024-11-20T22:34:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741870_1047 (size=175) 2024-11-20T22:34:49,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:49,615 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732142088957) 2024-11-20T22:34:49,638 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=407 (was 406) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48476 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48524 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:59892 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:49366 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=925 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=1872 (was 1255) - AvailableMemoryMB LEAK? - 2024-11-20T22:34:49,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=1872 2024-11-20T22:34:49,665 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:49,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:49,673 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732142089673 2024-11-20T22:34:49,714 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 2024-11-20T22:34:49,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:49,719 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a73fa9983b9429abb4c87237c19ff6b, NAME => 'testSequentialEditLogSeqNum,,1732142089665.7a73fa9983b9429abb4c87237c19ff6b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:49,719 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732142089665.7a73fa9983b9429abb4c87237c19ff6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:49,719 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,719 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,720 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b doesn't exist for region: 7a73fa9983b9429abb4c87237c19ff6b on table testSequentialEditLogSeqNum 2024-11-20T22:34:49,721 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 7a73fa9983b9429abb4c87237c19ff6b on table testSequentialEditLogSeqNum 2024-11-20T22:34:49,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741875_1052 (size=62) 2024-11-20T22:34:49,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741875_1052 (size=62) 2024-11-20T22:34:49,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741875_1052 (size=62) 2024-11-20T22:34:49,738 INFO [StoreOpener-7a73fa9983b9429abb4c87237c19ff6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,740 INFO [StoreOpener-7a73fa9983b9429abb4c87237c19ff6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a73fa9983b9429abb4c87237c19ff6b columnFamilyName a 2024-11-20T22:34:49,740 DEBUG [StoreOpener-7a73fa9983b9429abb4c87237c19ff6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:49,741 INFO [StoreOpener-7a73fa9983b9429abb4c87237c19ff6b-1 {}] regionserver.HStore(327): Store=7a73fa9983b9429abb4c87237c19ff6b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:49,741 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,743 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,743 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7a73fa9983b9429abb4c87237c19ff6b 2024-11-20T22:34:49,748 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:49,748 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7a73fa9983b9429abb4c87237c19ff6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60353816, jitterRate=-0.10065805912017822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:34:49,749 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7a73fa9983b9429abb4c87237c19ff6b: Writing region info on filesystem at 1732142089719Initializing all the Stores at 1732142089738 (+19 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142089738Cleaning up temporary data from old regions at 1732142089743 (+5 ms)Region opened successfully at 1732142089749 (+6 ms) 2024-11-20T22:34:49,780 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7a73fa9983b9429abb4c87237c19ff6b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-20T22:34:49,818 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/.tmp/a/d9e828d41827495ca8484f0cf58e3eec is 81, key is testSequentialEditLogSeqNum/a:x0/1732142089750/Put/seqid=0 2024-11-20T22:34:49,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741876_1053 (size=5833) 2024-11-20T22:34:49,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741876_1053 (size=5833) 2024-11-20T22:34:49,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741876_1053 (size=5833) 2024-11-20T22:34:50,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/.tmp/a/d9e828d41827495ca8484f0cf58e3eec 2024-11-20T22:34:50,256 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/.tmp/a/d9e828d41827495ca8484f0cf58e3eec as hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/a/d9e828d41827495ca8484f0cf58e3eec 2024-11-20T22:34:50,266 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/a/d9e828d41827495ca8484f0cf58e3eec, entries=10, sequenceid=13, filesize=5.7 K 2024-11-20T22:34:50,268 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7a73fa9983b9429abb4c87237c19ff6b in 488ms, sequenceid=13, compaction requested=false 2024-11-20T22:34:50,268 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7a73fa9983b9429abb4c87237c19ff6b: 2024-11-20T22:34:50,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T22:34:50,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T22:34:50,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T22:34:50,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T22:34:50,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T22:34:50,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741874_1051 (size=3256) 2024-11-20T22:34:50,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741874_1051 (size=3256) 2024-11-20T22:34:50,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741874_1051 (size=3256) 2024-11-20T22:34:50,300 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673, size=3.2 K (3256bytes) 2024-11-20T22:34:50,300 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 2024-11-20T22:34:50,301 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 after 1ms 2024-11-20T22:34:50,348 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:50,348 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 took 48ms 2024-11-20T22:34:50,355 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732142089673.temp 2024-11-20T22:34:50,356 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 so closing down 2024-11-20T22:34:50,356 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:50,398 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp 2024-11-20T22:34:50,398 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:50,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741877_1054 (size=2650) 2024-11-20T22:34:50,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741877_1054 (size=2650) 2024-11-20T22:34:50,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741877_1054 (size=2650) 2024-11-20T22:34:50,432 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp to hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000020 2024-11-20T22:34:50,432 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 83 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-20T22:34:50,432 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673, journal: Splitting hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673, size=3.2 K (3256bytes) at 1732142090300Finishing writing output for hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673 so closing down at 1732142090356 (+56 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp at 1732142090398 (+42 ms)3 split writer threads finished at 1732142090398Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732142090427 (+29 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000003-wal.1732142089673.temp to hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/7a73fa9983b9429abb4c87237c19ff6b/recovered.edits/0000000000000000020 at 1732142090432 (+5 ms)Processed 17 edits across 1 Regions in 83 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732142089664/wal.1732142089673, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732142090432 2024-11-20T22:34:50,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=410 (was 407) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48476 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48524 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:59892 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=961 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=1779 (was 1872) 2024-11-20T22:34:50,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410, OpenFileDescriptor=961, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=1770 2024-11-20T22:34:50,491 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:50,496 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:50,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:34:50,551 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91255916, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-91255916, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:50,570 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91255916/hregion-91255916.1732142090551, exclude list is [], retry=0 2024-11-20T22:34:50,575 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:50,580 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:50,580 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:50,634 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91255916/hregion-91255916.1732142090551 2024-11-20T22:34:50,636 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:50,637 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d33dc9d553adefb19cadfe72e62bf73, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:34:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741879_1056 (size=70) 2024-11-20T22:34:50,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741879_1056 (size=70) 2024-11-20T22:34:50,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741879_1056 (size=70) 2024-11-20T22:34:51,085 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:51,087 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,090 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName a 2024-11-20T22:34:51,090 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,096 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,097 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,102 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName b 2024-11-20T22:34:51,102 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,103 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,103 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,108 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName c 2024-11-20T22:34:51,108 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,109 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,109 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,111 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,111 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,112 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:51,114 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,119 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:51,120 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d33dc9d553adefb19cadfe72e62bf73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60562287, jitterRate=-0.09755159914493561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:51,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d33dc9d553adefb19cadfe72e62bf73: Writing region info on filesystem at 1732142091085Initializing all the Stores at 1732142091087 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091087Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091087Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091087Cleaning up temporary data from old regions at 1732142091111 (+24 ms)Region opened successfully at 1732142091121 (+10 ms) 2024-11-20T22:34:51,121 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d33dc9d553adefb19cadfe72e62bf73, disabling compactions & flushes 2024-11-20T22:34:51,121 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:51,122 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:51,122 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. after waiting 0 ms 2024-11-20T22:34:51,122 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:51,123 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:51,123 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d33dc9d553adefb19cadfe72e62bf73: Waiting for close lock at 1732142091121Disabling compacts and flushes for region at 1732142091121Disabling writes for close at 1732142091122 (+1 ms)Writing region close event to WAL at 1732142091122Closed at 1732142091122 2024-11-20T22:34:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741878_1055 (size=175) 2024-11-20T22:34:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741878_1055 (size=175) 2024-11-20T22:34:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741878_1055 (size=175) 2024-11-20T22:34:51,150 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:51,150 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91255916:(num 1732142090551) 2024-11-20T22:34:51,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:51,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:51,169 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, exclude list is [], retry=0 2024-11-20T22:34:51,173 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:51,174 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:51,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:51,212 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 2024-11-20T22:34:51,213 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:34:51,213 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d33dc9d553adefb19cadfe72e62bf73, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:51,213 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:51,213 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,213 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,215 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,216 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName a 2024-11-20T22:34:51,216 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,217 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,217 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,218 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName b 2024-11-20T22:34:51,218 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,219 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,219 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,220 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName c 2024-11-20T22:34:51,220 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:51,221 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:51,221 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,222 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,225 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:51,227 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:51,229 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d33dc9d553adefb19cadfe72e62bf73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59729037, jitterRate=-0.10996799170970917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:51,230 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d33dc9d553adefb19cadfe72e62bf73: Writing region info on filesystem at 1732142091213Initializing all the Stores at 1732142091214 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091214Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091215 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142091215Cleaning up temporary data from old regions at 1732142091225 (+10 ms)Region opened successfully at 1732142091230 (+5 ms) 2024-11-20T22:34:51,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732142091235/Put/seqid=0 2024-11-20T22:34:51,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741881_1058 (size=4826) 2024-11-20T22:34:51,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741881_1058 (size=4826) 2024-11-20T22:34:51,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741881_1058 (size=4826) 2024-11-20T22:34:51,255 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 0d33dc9d553adefb19cadfe72e62bf73/a 2024-11-20T22:34:51,272 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-20T22:34:51,276 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T22:34:51,277 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0d33dc9d553adefb19cadfe72e62bf73: 2024-11-20T22:34:51,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_ 2024-11-20T22:34:51,280 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0d33dc9d553adefb19cadfe72e62bf73/a as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_ - updating store file list. 2024-11-20T22:34:51,288 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T22:34:51,288 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_ into 0d33dc9d553adefb19cadfe72e62bf73/a 2024-11-20T22:34:51,288 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0d33dc9d553adefb19cadfe72e62bf73/a (new location: hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_) 2024-11-20T22:34:51,349 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, size=0 (0bytes) 2024-11-20T22:34:51,349 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 might be still open, length is 0 2024-11-20T22:34:51,349 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 2024-11-20T22:34:51,349 WARN [IPC Server handler 0 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-20T22:34:51,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 after 1ms 2024-11-20T22:34:53,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:59976 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59976 dst: /127.0.0.1:36643 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36643 remote=/127.0.0.1:59976]. Total timeout mills is 60000, 57776 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:53,522 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:48706 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48706 dst: /127.0.0.1:45551 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:53,522 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:49446 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37697:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49446 dst: /127.0.0.1:37697 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:53,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741880_1059 (size=606) 2024-11-20T22:34:53,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741880_1059 (size=606) 2024-11-20T22:34:55,351 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 after 4002ms 2024-11-20T22:34:55,400 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:55,400 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 took 4052ms 2024-11-20T22:34:55,404 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154; continuing. 2024-11-20T22:34:55,404 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 so closing down 2024-11-20T22:34:55,404 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:55,407 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732142091154.temp 2024-11-20T22:34:55,448 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp 2024-11-20T22:34:55,448 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:55,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741882_1060 (size=329) 2024-11-20T22:34:55,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741882_1060 (size=329) 2024-11-20T22:34:55,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741882_1060 (size=329) 2024-11-20T22:34:55,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T22:34:55,465 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp to hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 2024-11-20T22:34:55,465 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 63 ms; skipped=1; WAL=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T22:34:55,465 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, journal: Splitting hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, size=0 (0bytes) at 1732142091349Finishing writing output for hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 so closing down at 1732142095404 (+4055 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp at 1732142095448 (+44 ms)3 split writer threads finished at 1732142095448Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732142095463 (+15 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005-wal.1732142091154.temp to hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 at 1732142095465 (+2 ms)Processed 2 edits across 1 Regions in 63 ms; skipped=1; WAL=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154, size=0, length=0, corrupted=false, cancelled=false at 1732142095465 2024-11-20T22:34:55,467 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142091154 2024-11-20T22:34:55,469 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 2024-11-20T22:34:55,469 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:34:55,475 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:34:55,493 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142095475, exclude list is [], retry=0 2024-11-20T22:34:55,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:55,500 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:55,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:55,548 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142095475 2024-11-20T22:34:55,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:55,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d33dc9d553adefb19cadfe72e62bf73, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:55,553 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:55,553 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,553 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,556 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,558 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName a 2024-11-20T22:34:55,558 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:55,568 DEBUG [StoreFileOpener-0d33dc9d553adefb19cadfe72e62bf73-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T22:34:55,569 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/2abcf327bddb4b1bb66c0ae307ba764b_SeqId_3_ 2024-11-20T22:34:55,569 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:55,569 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,571 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName b 2024-11-20T22:34:55,571 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:55,571 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:55,572 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,573 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d33dc9d553adefb19cadfe72e62bf73 columnFamilyName c 2024-11-20T22:34:55,573 DEBUG [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:55,574 INFO [StoreOpener-0d33dc9d553adefb19cadfe72e62bf73-1 {}] regionserver.HStore(327): Store=0d33dc9d553adefb19cadfe72e62bf73/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:55,574 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,580 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 2024-11-20T22:34:55,628 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:55,630 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 2024-11-20T22:34:55,630 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d33dc9d553adefb19cadfe72e62bf73 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-20T22:34:55,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/.tmp/a/c1f8083061404ac4b82964e91db6d75b is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732142091296/Put/seqid=0 2024-11-20T22:34:55,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741884_1062 (size=5149) 2024-11-20T22:34:55,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741884_1062 (size=5149) 2024-11-20T22:34:55,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741884_1062 (size=5149) 2024-11-20T22:34:55,659 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/.tmp/a/c1f8083061404ac4b82964e91db6d75b 2024-11-20T22:34:55,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/.tmp/a/c1f8083061404ac4b82964e91db6d75b as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/c1f8083061404ac4b82964e91db6d75b 2024-11-20T22:34:55,672 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/a/c1f8083061404ac4b82964e91db6d75b, entries=1, sequenceid=5, filesize=5.0 K 2024-11-20T22:34:55,672 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0d33dc9d553adefb19cadfe72e62bf73 in 42ms, sequenceid=5, compaction requested=false; wal=null 2024-11-20T22:34:55,673 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/0000000000000000005 2024-11-20T22:34:55,674 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,674 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,675 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:34:55,677 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d33dc9d553adefb19cadfe72e62bf73 2024-11-20T22:34:55,680 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0d33dc9d553adefb19cadfe72e62bf73/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-20T22:34:55,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d33dc9d553adefb19cadfe72e62bf73; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66522117, jitterRate=-0.00874321162700653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:34:55,682 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d33dc9d553adefb19cadfe72e62bf73: Writing region info on filesystem at 1732142095554Initializing all the Stores at 1732142095555 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142095555Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142095556 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142095556Obtaining lock to block concurrent updates at 1732142095630 (+74 ms)Preparing flush snapshotting stores in 0d33dc9d553adefb19cadfe72e62bf73 at 1732142095630Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732142095630Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. at 1732142095630Flushing 0d33dc9d553adefb19cadfe72e62bf73/a: creating writer at 1732142095631 (+1 ms)Flushing 0d33dc9d553adefb19cadfe72e62bf73/a: appending metadata at 1732142095646 (+15 ms)Flushing 0d33dc9d553adefb19cadfe72e62bf73/a: closing flushed file at 1732142095646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0cda9d: reopening flushed file at 1732142095665 (+19 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0d33dc9d553adefb19cadfe72e62bf73 in 42ms, sequenceid=5, compaction requested=false; wal=null at 1732142095672 (+7 ms)Cleaning up temporary data from old regions at 1732142095674 (+2 ms)Region opened successfully at 1732142095682 (+8 ms) 2024-11-20T22:34:55,687 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d33dc9d553adefb19cadfe72e62bf73, disabling compactions & flushes 2024-11-20T22:34:55,687 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:55,687 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:55,687 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. after waiting 0 ms 2024-11-20T22:34:55,687 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:55,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732142090494.0d33dc9d553adefb19cadfe72e62bf73. 2024-11-20T22:34:55,689 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d33dc9d553adefb19cadfe72e62bf73: Waiting for close lock at 1732142095687Disabling compacts and flushes for region at 1732142095687Disabling writes for close at 1732142095687Writing region close event to WAL at 1732142095688 (+1 ms)Closed at 1732142095688 2024-11-20T22:34:55,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741883_1061 (size=175) 2024-11-20T22:34:55,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741883_1061 (size=175) 2024-11-20T22:34:55,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741883_1061 (size=175) 2024-11-20T22:34:55,695 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:34:55,696 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732142095475) 2024-11-20T22:34:55,710 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=416 (was 410) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1113328909_22 at /127.0.0.1:38268 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1113328909_22 at /127.0.0.1:56356 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46165 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1113328909_22 at /127.0.0.1:35770 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 961) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=1270 (was 1770) 2024-11-20T22:34:55,712 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-20T22:34:55,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=416, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=1269 2024-11-20T22:34:55,724 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-20T22:34:55,740 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:55,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T22:34:55,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f31b4695a16,40769,1732142076492 2024-11-20T22:34:55,749 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d27917 2024-11-20T22:34:55,750 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T22:34:55,752 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T22:34:55,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T22:34:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T22:34:55,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T22:34:55,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-20T22:34:55,765 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:55,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T22:34:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T22:34:55,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741885_1063 (size=694) 2024-11-20T22:34:55,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741885_1063 (size=694) 2024-11-20T22:34:55,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741885_1063 (size=694) 2024-11-20T22:34:55,780 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647 2024-11-20T22:34:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741886_1064 (size=77) 2024-11-20T22:34:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741886_1064 (size=77) 2024-11-20T22:34:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741886_1064 (size=77) 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:55,790 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:55,790 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:55,790 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142095790Disabling compacts and flushes for region at 1732142095790Disabling writes for close at 1732142095790Writing region close event to WAL at 1732142095790Closed at 1732142095790 2024-11-20T22:34:55,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T22:34:55,802 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732142095795"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732142095795"}]},"ts":"1732142095795"} 2024-11-20T22:34:55,805 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T22:34:55,807 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T22:34:55,810 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732142095807"}]},"ts":"1732142095807"} 2024-11-20T22:34:55,814 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-20T22:34:55,815 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9f31b4695a16=0} racks are {/default-rack=0} 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T22:34:55,816 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T22:34:55,816 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T22:34:55,816 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T22:34:55,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T22:34:55,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN}] 2024-11-20T22:34:55,820 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN 2024-11-20T22:34:55,822 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN; state=OFFLINE, location=9f31b4695a16,43093,1732142077335; forceNewPlan=false, retain=false 2024-11-20T22:34:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T22:34:55,974 INFO [9f31b4695a16:40769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T22:34:55,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPENING, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:55,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN because future has completed 2024-11-20T22:34:55,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335}] 2024-11-20T22:34:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T22:34:56,144 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,145 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:56,145 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,145 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:56,146 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,146 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,148 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,150 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:34:56,151 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:56,151 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:56,151 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,153 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:34:56,153 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:56,154 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:56,154 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,155 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,156 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,156 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,156 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,157 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T22:34:56,159 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,165 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:34:56,166 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5e972e424facefaa32be2f83dd0a8f47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63880278, jitterRate=-0.04810968041419983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T22:34:56,166 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,167 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5e972e424facefaa32be2f83dd0a8f47: Running coprocessor pre-open hook at 1732142096146Writing region info on filesystem at 1732142096146Initializing all the Stores at 1732142096147 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142096147Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142096148 (+1 ms)Cleaning up temporary data from old regions at 1732142096156 (+8 ms)Running coprocessor post-open hooks at 1732142096166 (+10 ms)Region opened successfully at 1732142096167 (+1 ms) 2024-11-20T22:34:56,169 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., pid=6, masterSystemTime=1732142096133 2024-11-20T22:34:56,172 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,172 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,173 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPEN, openSeqNum=2, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:56,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 because future has completed 2024-11-20T22:34:56,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T22:34:56,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 in 201 msec 2024-11-20T22:34:56,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T22:34:56,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN in 366 msec 2024-11-20T22:34:56,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T22:34:56,189 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732142096189"}]},"ts":"1732142096189"} 2024-11-20T22:34:56,193 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-20T22:34:56,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T22:34:56,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 439 msec 2024-11-20T22:34:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T22:34:56,403 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-20T22:34:56,403 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-20T22:34:56,404 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T22:34:56,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-20T22:34:56,413 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T22:34:56,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-20T22:34:56,426 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=2] 2024-11-20T22:34:56,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5e972e424facefaa32be2f83dd0a8f47, source=9f31b4695a16,43093,1732142077335, destination=9f31b4695a16,43815,1732142077481, warming up region on 9f31b4695a16,43815,1732142077481 2024-11-20T22:34:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T22:34:56,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5e972e424facefaa32be2f83dd0a8f47, source=9f31b4695a16,43093,1732142077335, destination=9f31b4695a16,43815,1732142077481, running balancer 2024-11-20T22:34:56,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55359, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T22:34:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE 2024-11-20T22:34:56,451 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE 2024-11-20T22:34:56,454 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=CLOSING, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:56,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:56,456 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,458 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:34:56,458 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:56,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE because future has completed 2024-11-20T22:34:56,460 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T22:34:56,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335}] 2024-11-20T22:34:56,462 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:56,462 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,465 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:34:56,465 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:56,472 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:56,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142096472Disabling compacts and flushes for region at 1732142096472Disabling writes for close at 1732142096473 (+1 ms)Writing region close event to WAL at 1732142096477 (+4 ms)Closed at 1732142096477 2024-11-20T22:34:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T22:34:56,621 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,622 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T22:34:56,622 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:56,623 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,623 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,623 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:56,623 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,623 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 5e972e424facefaa32be2f83dd0a8f47 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-20T22:34:56,647 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/8f06fbf22a9f41f89c41c8e51fff6075 is 35, key is r1/cf1:q/1732142096429/Put/seqid=0 2024-11-20T22:34:56,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741887_1065 (size=4783) 2024-11-20T22:34:56,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741887_1065 (size=4783) 2024-11-20T22:34:56,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741887_1065 (size=4783) 2024-11-20T22:34:56,673 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/8f06fbf22a9f41f89c41c8e51fff6075 2024-11-20T22:34:56,682 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/8f06fbf22a9f41f89c41c8e51fff6075 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075 2024-11-20T22:34:56,689 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075, entries=1, sequenceid=5, filesize=4.7 K 2024-11-20T22:34:56,691 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5e972e424facefaa32be2f83dd0a8f47 in 68ms, sequenceid=5, compaction requested=false 2024-11-20T22:34:56,691 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-20T22:34:56,696 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T22:34:56,698 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:56,698 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142096622Running coprocessor pre-close hooks at 1732142096622Disabling compacts and flushes for region at 1732142096622Disabling writes for close at 1732142096623 (+1 ms)Obtaining lock to block concurrent updates at 1732142096623Preparing flush snapshotting stores in 5e972e424facefaa32be2f83dd0a8f47 at 1732142096623Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732142096623Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. at 1732142096624 (+1 ms)Flushing 5e972e424facefaa32be2f83dd0a8f47/cf1: creating writer at 1732142096624Flushing 5e972e424facefaa32be2f83dd0a8f47/cf1: appending metadata at 1732142096646 (+22 ms)Flushing 5e972e424facefaa32be2f83dd0a8f47/cf1: closing flushed file at 1732142096647 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5558c279: reopening flushed file at 1732142096681 (+34 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5e972e424facefaa32be2f83dd0a8f47 in 68ms, sequenceid=5, compaction requested=false at 1732142096691 (+10 ms)Writing region close event to WAL at 1732142096692 (+1 ms)Running coprocessor post-close hooks at 1732142096696 (+4 ms)Closed at 1732142096698 (+2 ms) 2024-11-20T22:34:56,699 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5e972e424facefaa32be2f83dd0a8f47 move to 9f31b4695a16,43815,1732142077481 record at close sequenceid=5 2024-11-20T22:34:56,702 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:56,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=CLOSED 2024-11-20T22:34:56,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 because future has completed 2024-11-20T22:34:56,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T22:34:56,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 in 249 msec 2024-11-20T22:34:56,717 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE; state=CLOSED, location=9f31b4695a16,43815,1732142077481; forceNewPlan=false, retain=false 2024-11-20T22:34:56,849 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-20T22:34:56,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-20T22:34:56,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-20T22:34:56,851 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-20T22:34:56,867 INFO [9f31b4695a16:40769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T22:34:56,868 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPENING, regionLocation=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:56,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE because future has completed 2024-11-20T22:34:56,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481}] 2024-11-20T22:34:57,031 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,031 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:57,031 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,031 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:57,031 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,031 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,033 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,034 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:34:57,035 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:57,042 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075 2024-11-20T22:34:57,042 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:57,042 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,043 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:34:57,043 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:57,043 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:57,044 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,044 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,045 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,046 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,046 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,047 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T22:34:57,048 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,049 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5e972e424facefaa32be2f83dd0a8f47; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73358464, jitterRate=0.09312629699707031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T22:34:57,050 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,050 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5e972e424facefaa32be2f83dd0a8f47: Running coprocessor pre-open hook at 1732142097031Writing region info on filesystem at 1732142097031Initializing all the Stores at 1732142097033 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142097033Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142097033Cleaning up temporary data from old regions at 1732142097046 (+13 ms)Running coprocessor post-open hooks at 1732142097050 (+4 ms)Region opened successfully at 1732142097050 2024-11-20T22:34:57,052 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., pid=9, masterSystemTime=1732142097026 2024-11-20T22:34:57,054 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,055 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,056 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPEN, openSeqNum=9, regionLocation=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:57,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481 because future has completed 2024-11-20T22:34:57,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T22:34:57,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481 in 188 msec 2024-11-20T22:34:57,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE in 614 msec 2024-11-20T22:34:57,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T22:34:57,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T22:34:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:48140 deadline: 1732142157116, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43815 startCode=1732142077481. As of locationSeqNum=5. 2024-11-20T22:34:57,141 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43815 startCode=1732142077481. As of locationSeqNum=5. 2024-11-20T22:34:57,141 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43815 startCode=1732142077481. As of locationSeqNum=5. 2024-11-20T22:34:57,142 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43815,1732142077481, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43815 startCode=1732142077481. As of locationSeqNum=5. 2024-11-20T22:34:57,250 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T22:34:57,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T22:34:57,262 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5e972e424facefaa32be2f83dd0a8f47 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-20T22:34:57,280 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/56723843d92246afa935a71f014a7b4c is 29, key is r1/cf1:/1732142097254/DeleteFamily/seqid=0 2024-11-20T22:34:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741888_1066 (size=4906) 2024-11-20T22:34:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741888_1066 (size=4906) 2024-11-20T22:34:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741888_1066 (size=4906) 2024-11-20T22:34:57,291 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,299 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,316 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf2/98fc1b744abc45eb9f318a8c5be32a17 is 29, key is r1/cf2:/1732142097254/DeleteFamily/seqid=0 2024-11-20T22:34:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741889_1067 (size=4906) 2024-11-20T22:34:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741889_1067 (size=4906) 2024-11-20T22:34:57,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741889_1067 (size=4906) 2024-11-20T22:34:57,328 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf2/98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,335 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/56723843d92246afa935a71f014a7b4c as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,344 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,344 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T22:34:57,346 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf2/98fc1b744abc45eb9f318a8c5be32a17 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,353 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,353 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T22:34:57,354 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 5e972e424facefaa32be2f83dd0a8f47 in 92ms, sequenceid=12, compaction requested=false 2024-11-20T22:34:57,355 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5e972e424facefaa32be2f83dd0a8f47: 2024-11-20T22:34:57,358 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T22:34:57,360 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5e972e424facefaa32be2f83dd0a8f47/cf1 is initiating major compaction (all files) 2024-11-20T22:34:57,360 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T22:34:57,360 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:57,360 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5e972e424facefaa32be2f83dd0a8f47/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,361 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075, hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c] into tmpdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp, totalSize=9.5 K 2024-11-20T22:34:57,362 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8f06fbf22a9f41f89c41c8e51fff6075, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732142096429 2024-11-20T22:34:57,363 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 56723843d92246afa935a71f014a7b4c, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T22:34:57,378 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5e972e424facefaa32be2f83dd0a8f47#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T22:34:57,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741890_1068 (size=4626) 2024-11-20T22:34:57,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741890_1068 (size=4626) 2024-11-20T22:34:57,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741890_1068 (size=4626) 2024-11-20T22:34:57,396 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf1/a7ef913c2c2b46508600f57b64591dff as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/a7ef913c2c2b46508600f57b64591dff 2024-11-20T22:34:57,414 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 5e972e424facefaa32be2f83dd0a8f47/cf1 of 5e972e424facefaa32be2f83dd0a8f47 into a7ef913c2c2b46508600f57b64591dff(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T22:34:57,415 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5e972e424facefaa32be2f83dd0a8f47: 2024-11-20T22:34:57,415 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-20T22:34:57,415 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5e972e424facefaa32be2f83dd0a8f47/cf2 is initiating major compaction (all files) 2024-11-20T22:34:57,415 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T22:34:57,415 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T22:34:57,415 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5e972e424facefaa32be2f83dd0a8f47/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,416 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17] into tmpdir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp, totalSize=4.8 K 2024-11-20T22:34:57,416 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 98fc1b744abc45eb9f318a8c5be32a17, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T22:34:57,429 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5e972e424facefaa32be2f83dd0a8f47#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T22:34:57,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741891_1069 (size=4592) 2024-11-20T22:34:57,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741891_1069 (size=4592) 2024-11-20T22:34:57,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741891_1069 (size=4592) 2024-11-20T22:34:57,467 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/.tmp/cf2/14f0bfd33bc844e1aa72976291f33410 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/14f0bfd33bc844e1aa72976291f33410 2024-11-20T22:34:57,476 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 5e972e424facefaa32be2f83dd0a8f47/cf2 of 5e972e424facefaa32be2f83dd0a8f47 into 14f0bfd33bc844e1aa72976291f33410(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T22:34:57,477 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5e972e424facefaa32be2f83dd0a8f47: 2024-11-20T22:34:57,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5e972e424facefaa32be2f83dd0a8f47, source=9f31b4695a16,43815,1732142077481, destination=9f31b4695a16,43093,1732142077335, warming up region on 9f31b4695a16,43093,1732142077335 2024-11-20T22:34:57,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5e972e424facefaa32be2f83dd0a8f47, source=9f31b4695a16,43815,1732142077481, destination=9f31b4695a16,43093,1732142077335, running balancer 2024-11-20T22:34:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE 2024-11-20T22:34:57,485 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE 2024-11-20T22:34:57,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:57,487 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=CLOSING, regionLocation=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:57,487 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,489 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:34:57,489 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:57,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE because future has completed 2024-11-20T22:34:57,491 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T22:34:57,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481}] 2024-11-20T22:34:57,500 INFO [StoreFileOpener-5e972e424facefaa32be2f83dd0a8f47-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,500 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,508 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075 2024-11-20T22:34:57,513 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/a7ef913c2c2b46508600f57b64591dff 2024-11-20T22:34:57,513 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:57,514 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,515 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:34:57,515 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:57,524 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/14f0bfd33bc844e1aa72976291f33410 2024-11-20T22:34:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-20T22:34:57,538 INFO [StoreFileOpener-5e972e424facefaa32be2f83dd0a8f47-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,538 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,538 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:57,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43093 {}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142097538Disabling compacts and flushes for region at 1732142097538Disabling writes for close at 1732142097539 (+1 ms)Writing region close event to WAL at 1732142097544 (+5 ms)Closed at 1732142097544 2024-11-20T22:34:57,648 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,649 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T22:34:57,649 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:57,649 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,649 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,649 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:57,649 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,650 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075, hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c] to archive 2024-11-20T22:34:57,654 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T22:34:57,659 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075 to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/8f06fbf22a9f41f89c41c8e51fff6075 2024-11-20T22:34:57,662 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/56723843d92246afa935a71f014a7b4c 2024-11-20T22:34:57,681 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17] to archive 2024-11-20T22:34:57,682 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T22:34:57,686 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17 to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/98fc1b744abc45eb9f318a8c5be32a17 2024-11-20T22:34:57,692 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-20T22:34:57,693 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:57,693 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142097649Running coprocessor pre-close hooks at 1732142097649Disabling compacts and flushes for region at 1732142097649Disabling writes for close at 1732142097649Writing region close event to WAL at 1732142097688 (+39 ms)Running coprocessor post-close hooks at 1732142097693 (+5 ms)Closed at 1732142097693 2024-11-20T22:34:57,693 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 5e972e424facefaa32be2f83dd0a8f47 move to 9f31b4695a16,43093,1732142077335 record at close sequenceid=12 2024-11-20T22:34:57,696 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:57,699 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=CLOSED 2024-11-20T22:34:57,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481 because future has completed 2024-11-20T22:34:57,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T22:34:57,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43815,1732142077481 in 246 msec 2024-11-20T22:34:57,743 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE; state=CLOSED, location=9f31b4695a16,43093,1732142077335; forceNewPlan=false, retain=false 2024-11-20T22:34:57,894 INFO [9f31b4695a16:40769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T22:34:57,895 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPENING, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:57,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE because future has completed 2024-11-20T22:34:57,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335}] 2024-11-20T22:34:58,058 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,058 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:58,059 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,059 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:58,059 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,059 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,061 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,062 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:34:58,062 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:58,069 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/a7ef913c2c2b46508600f57b64591dff 2024-11-20T22:34:58,069 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:58,070 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,071 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:34:58,071 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:58,079 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/14f0bfd33bc844e1aa72976291f33410 2024-11-20T22:34:58,079 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:34:58,079 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,080 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,082 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,083 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,083 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,084 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T22:34:58,086 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,087 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 5e972e424facefaa32be2f83dd0a8f47; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66077883, jitterRate=-0.01536281406879425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T22:34:58,088 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,088 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5e972e424facefaa32be2f83dd0a8f47: Running coprocessor pre-open hook at 1732142098059Writing region info on filesystem at 1732142098059Initializing all the Stores at 1732142098060 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142098060Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142098061 (+1 ms)Cleaning up temporary data from old regions at 1732142098083 (+22 ms)Running coprocessor post-open hooks at 1732142098088 (+5 ms)Region opened successfully at 1732142098088 2024-11-20T22:34:58,089 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., pid=12, masterSystemTime=1732142098053 2024-11-20T22:34:58,092 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,092 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,093 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPEN, openSeqNum=18, regionLocation=9f31b4695a16,43093,1732142077335 2024-11-20T22:34:58,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 because future has completed 2024-11-20T22:34:58,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-20T22:34:58,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,43093,1732142077335 in 197 msec 2024-11-20T22:34:58,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, REOPEN/MOVE in 617 msec 2024-11-20T22:34:58,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T22:34:58,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T22:34:58,141 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9f31b4695a16,43093,1732142077335: testing ***** 2024-11-20T22:34:58,142 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T22:34:58,143 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T22:34:58,145 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T22:34:58,151 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T22:34:58,153 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T22:34:58,164 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 496633912 }, "NonHeapMemoryUsage": { "committed": 172359680, "init": 7667712, "max": -1, "used": 169702960 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9f31b4695a16", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 62, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 27, "ProcessCallTime_98th_percentile": 62, "ProcessCallTime_99th_percentile": 62, "ProcessCallTime_99.9th_percentile": 62, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 15, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 2, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 62, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 27, "TotalCallTime_98th_percentile": 62, "TotalCallTime_99th_percentile": 62, "TotalCallTime_99.9th_percentile": 62, "TotalCallTime_TimeRangeCount_0-1": 7, "TotalCallTime_TimeRangeCount_1-3": 17, "TotalCallTime_TimeRangeCount_3-10": 4, "TotalCallTime_TimeRangeCount_10-30": 2, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5781 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9f31b4695a16", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62486", "tag.serverName": "9f31b4695a16,43093,1732142077335", "tag.clusterId": "6811c5ba-3839-4753-8916-d16cd6b50586", "tag.Context": "regionserver", "tag.Hostname": "9f31b4695a16", "regionCount": 2, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732142077335, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 3, "ScanTime_TimeRangeCount_1-3": 2, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 1, "Put_max": 14, "Put_mean": 2, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 14, "Put_98th_percentile": 14, "Put_99th_percentile": 14, "Put_99.9th_percentile": 14, "Put_TimeRangeCount_0-1": 4, "Put_TimeRangeCount_1-3": 9, "Put_TimeRangeCount_10-30": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 68, "FlushTime_max": 68, "FlushTime_mean": 68, "FlushTime_25th_percentile": 68, "FlushTime_median": 68, "FlushTime_75th_percentile": 68, "FlushTime_90th_percentile": 68, "FlushTime_95th_percentile": 68, "FlushTime_98th_percentile": 68, "FlushTime_99th_percentile": 68, "FlushTime_99.9th_percentile": 68, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T22:34:58,167 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40769 {}] master.MasterRpcServices(700): 9f31b4695a16,43093,1732142077335 reported a fatal error: ***** ABORTING region server 9f31b4695a16,43093,1732142077335: testing ***** 2024-11-20T22:34:58,170 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9f31b4695a16,43093,1732142077335' ***** 2024-11-20T22:34:58,170 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-20T22:34:58,170 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T22:34:58,170 INFO [RS:0;9f31b4695a16:43093 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T22:34:58,170 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T22:34:58,170 INFO [RS:0;9f31b4695a16:43093 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T22:34:58,171 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,171 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(956): aborting server 9f31b4695a16,43093,1732142077335 2024-11-20T22:34:58,171 INFO [RS:0;9f31b4695a16:43093 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T22:34:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43815 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:43454 deadline: 1732142158171, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43093 startCode=1732142077335. As of locationSeqNum=12. 2024-11-20T22:34:58,171 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:34:58,171 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,171 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,171 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:34:58,172 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,172 INFO [RS:0;9f31b4695a16:43093 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9f31b4695a16:43093. 2024-11-20T22:34:58,172 DEBUG [RS:0;9f31b4695a16:43093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T22:34:58,172 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43815,1732142077481, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43815,1732142077481, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43093 startCode=1732142077335. As of locationSeqNum=12. 2024-11-20T22:34:58,172 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43815,1732142077481, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43093 startCode=1732142077335. As of locationSeqNum=12. 2024-11-20T22:34:58,172 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43815,1732142077481, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9f31b4695a16 port=43093 startCode=1732142077335. As of locationSeqNum=12. 2024-11-20T22:34:58,172 DEBUG [RS:0;9f31b4695a16:43093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:34:58,173 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T22:34:58,173 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T22:34:58,173 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T22:34:58,173 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T22:34:58,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:58,175 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting 2024-11-20T22:34:58,176 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting 2024-11-20T22:34:58,176 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 from cache 2024-11-20T22:34:58,176 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T22:34:58,177 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1325): Online Regions={5e972e424facefaa32be2f83dd0a8f47=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T22:34:58,177 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T22:34:58,177 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T22:34:58,177 DEBUG [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:34:58,177 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,177 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142098171Running coprocessor pre-close hooks at 1732142098171Disabling compacts and flushes for region at 1732142098171Disabling writes for close at 1732142098171Writing region close event to WAL at 1732142098177 (+6 ms)Running coprocessor post-close hooks at 1732142098177Closed at 1732142098177 2024-11-20T22:34:58,177 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T22:34:58,178 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:34:58,177 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T22:34:58,178 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T22:34:58,180 ERROR [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-20T22:34:58,181 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T22:34:58,181 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T22:34:58,181 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732142098177Running coprocessor pre-close hooks at 1732142098177Disabling compacts and flushes for region at 1732142098177Disabling writes for close at 1732142098178 (+1 ms)Writing region close event to WAL at 1732142098180 (+2 ms)Running coprocessor post-close hooks at 1732142098180Closed at 1732142098181 (+1 ms) 2024-11-20T22:34:58,181 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T22:34:58,239 INFO [regionserver/9f31b4695a16:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:34:58,280 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:58,281 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1] 2024-11-20T22:34:58,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting 2024-11-20T22:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9f31b4695a16,43093,1732142077335 aborting 2024-11-20T22:34:58,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 from cache 2024-11-20T22:34:58,378 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(976): stopping server 9f31b4695a16,43093,1732142077335; all regions closed. 2024-11-20T22:34:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741836_1012 (size=7323) 2024-11-20T22:34:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741836_1012 (size=7323) 2024-11-20T22:34:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741836_1012 (size=7323) 2024-11-20T22:34:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741835_1011 (size=2288) 2024-11-20T22:34:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741835_1011 (size=2288) 2024-11-20T22:34:58,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741835_1011 (size=2288) 2024-11-20T22:34:58,385 DEBUG [RS:0;9f31b4695a16:43093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:34:58,385 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:34:58,386 INFO [RS:0;9f31b4695a16:43093 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T22:34:58,386 INFO [RS:0;9f31b4695a16:43093 {}] hbase.ChoreService(370): Chore service for: regionserver/9f31b4695a16:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T22:34:58,386 INFO [RS:0;9f31b4695a16:43093 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T22:34:58,386 INFO [regionserver/9f31b4695a16:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T22:34:58,387 INFO [RS:0;9f31b4695a16:43093 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43093 2024-11-20T22:34:58,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T22:34:58,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9f31b4695a16,43093,1732142077335 2024-11-20T22:34:58,402 INFO [RS:0;9f31b4695a16:43093 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T22:34:58,403 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007fa8848e28e8@28d7e63a rejected from java.util.concurrent.ThreadPoolExecutor@6dd56d44[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 12] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-20T22:34:58,419 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9f31b4695a16,43093,1732142077335] 2024-11-20T22:34:58,439 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9f31b4695a16,43093,1732142077335 already deleted, retry=false 2024-11-20T22:34:58,440 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9f31b4695a16,43093,1732142077335 on 9f31b4695a16,40769,1732142076492 2024-11-20T22:34:58,445 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9f31b4695a16,43093,1732142077335, splitWal=true, meta=true 2024-11-20T22:34:58,448 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 9f31b4695a16,43093,1732142077335 (carryingMeta=true) 9f31b4695a16,43093,1732142077335/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@51d324c9[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T22:34:58,448 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9f31b4695a16,43093,1732142077335, splitWal=true, meta=true 2024-11-20T22:34:58,450 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9f31b4695a16,43093,1732142077335, splitWal=true, meta=true, isMeta: true 2024-11-20T22:34:58,452 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting 2024-11-20T22:34:58,454 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9f31b4695a16,43093,1732142077335 WAL count=1, meta=true 2024-11-20T22:34:58,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta}] 2024-11-20T22:34:58,463 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:58,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, worker=9f31b4695a16,43815,1732142077481}] 2024-11-20T22:34:58,490 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:58,491 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1] 2024-11-20T22:34:58,493 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9f31b4695a16:43093 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9f31b4695a16/172.17.0.2:43093 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:58,494 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1, error=java.net.ConnectException: Call to address=9f31b4695a16:43093 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9f31b4695a16/172.17.0.2:43093 2024-11-20T22:34:58,494 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-20T22:34:58,494 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 from cache 2024-11-20T22:34:58,495 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 9f31b4695a16:43093 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9f31b4695a16/172.17.0.2:43093 2024-11-20T22:34:58,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:34:58,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43093-0x1015a628ac10001, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:34:58,520 INFO [RS:0;9f31b4695a16:43093 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T22:34:58,520 INFO [RS:0;9f31b4695a16:43093 {}] regionserver.HRegionServer(1031): Exiting; stopping=9f31b4695a16,43093,1732142077335; zookeeper connection closed. 2024-11-20T22:34:58,521 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c8ffa21 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c8ffa21 2024-11-20T22:34:58,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43815 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-20T22:34:58,651 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, size=7.2 K (7323bytes) 2024-11-20T22:34:58,652 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta 2024-11-20T22:34:58,652 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta after 0ms 2024-11-20T22:34:58,685 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:58,686 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta took 34ms 2024-11-20T22:34:58,693 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T22:34:58,694 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta so closing down 2024-11-20T22:34:58,694 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:58,698 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp 2024-11-20T22:34:58,725 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp 2024-11-20T22:34:58,726 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:58,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741892_1070 (size=7071) 2024-11-20T22:34:58,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741892_1070 (size=7071) 2024-11-20T22:34:58,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741892_1070 (size=7071) 2024-11-20T22:34:58,745 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp (wrote 15 edits, skipped 0 edits in 1 ms) 2024-11-20T22:34:58,746 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T22:34:58,747 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-20T22:34:58,747 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, journal: Splitting hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, size=7.2 K (7323bytes) at 1732142098652Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta so closing down at 1732142098694 (+42 ms)Creating recovered edits writer path=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp at 1732142098725 (+31 ms)3 split writer threads finished at 1732142098726 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp (wrote 15 edits, skipped 0 edits in 1 ms) at 1732142098745 (+19 ms)Rename recovered edits hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta.temp to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732142098746 (+1 ms)Processed 16 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1732142098747 (+1 ms) 2024-11-20T22:34:58,747 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta 2024-11-20T22:34:58,748 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-20T22:34:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-20T22:34:58,756 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs 2024-11-20T22:34:58,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T22:34:58,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, worker=9f31b4695a16,43815,1732142077481 in 291 msec 2024-11-20T22:34:58,762 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=9f31b4695a16,43815,1732142077481 2024-11-20T22:34:58,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T22:34:58,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9f31b4695a16%2C43093%2C1732142077335.meta.1732142079957.meta, worker=9f31b4695a16,43815,1732142077481 in 307 msec 2024-11-20T22:34:58,769 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting dir is empty, no logs to split. 2024-11-20T22:34:58,770 INFO [PEWorker-5 {}] master.SplitWALManager(105): 9f31b4695a16,43093,1732142077335 WAL count=0, meta=true 2024-11-20T22:34:58,770 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 9f31b4695a16,43093,1732142077335 WAL splitting is done? wals=0, meta=true 2024-11-20T22:34:58,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T22:34:58,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T22:34:58,778 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T22:34:58,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:58,801 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1] 2024-11-20T22:34:58,802 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9f31b4695a16:43093 this server is in the failed servers list 2024-11-20T22:34:58,803 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9f31b4695a16:43093 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9f31b4695a16:43093 2024-11-20T22:34:58,803 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9f31b4695a16:43093 2024-11-20T22:34:58,803 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 from cache 2024-11-20T22:34:58,928 DEBUG [9f31b4695a16:40769 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(204): Hosts are {9f31b4695a16=0} racks are {/default-rack=0} 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T22:34:58,929 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T22:34:58,929 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T22:34:58,929 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T22:34:58,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:58,931 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f31b4695a16,36989,1732142077539, state=OPENING 2024-11-20T22:34:58,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:58,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:58,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:58,939 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T22:34:58,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:58,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:58,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:58,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9f31b4695a16,36989,1732142077539}] 2024-11-20T22:34:59,092 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T22:34:59,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T22:34:59,100 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T22:34:59,100 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:34:59,101 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T22:34:59,103 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f31b4695a16%2C36989%2C1732142077539.meta, suffix=.meta, logDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs, maxLogs=32 2024-11-20T22:34:59,117 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539/9f31b4695a16%2C36989%2C1732142077539.meta.1732142099104.meta, exclude list is [], retry=0 2024-11-20T22:34:59,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:34:59,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:34:59,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:34:59,145 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,36989,1732142077539/9f31b4695a16%2C36989%2C1732142077539.meta.1732142099104.meta 2024-11-20T22:34:59,146 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:34:59,146 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:34:59,146 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T22:34:59,146 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T22:34:59,146 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T22:34:59,146 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T22:34:59,147 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:34:59,147 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T22:34:59,147 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T22:34:59,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T22:34:59,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T22:34:59,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:59,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:59,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T22:34:59,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T22:34:59,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:59,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:59,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T22:34:59,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T22:34:59,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:59,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:59,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T22:34:59,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T22:34:59,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:34:59,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T22:34:59,155 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T22:34:59,156 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:59,157 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740 2024-11-20T22:34:59,158 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T22:34:59,185 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:59,188 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T22:34:59,189 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-20T22:34:59,214 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/ec39b0e9bc5a45c0a939bb0a545371e4 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47./info:regioninfo/1732142098093/Put/seqid=0 2024-11-20T22:34:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741894_1072 (size=11177) 2024-11-20T22:34:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741894_1072 (size=11177) 2024-11-20T22:34:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741894_1072 (size=11177) 2024-11-20T22:34:59,221 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/ec39b0e9bc5a45c0a939bb0a545371e4 2024-11-20T22:34:59,249 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/ns/abdd585f6c60486fbf4629ab7ecdd9c0 is 43, key is default/ns:d/1732142080250/Put/seqid=0 2024-11-20T22:34:59,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741895_1073 (size=5153) 2024-11-20T22:34:59,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741895_1073 (size=5153) 2024-11-20T22:34:59,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741895_1073 (size=5153) 2024-11-20T22:34:59,259 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/ns/abdd585f6c60486fbf4629ab7ecdd9c0 2024-11-20T22:34:59,283 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/table/2514927885324fe2906a075373c8f051 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732142096189/Put/seqid=0 2024-11-20T22:34:59,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741896_1074 (size=5431) 2024-11-20T22:34:59,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741896_1074 (size=5431) 2024-11-20T22:34:59,296 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/table/2514927885324fe2906a075373c8f051 2024-11-20T22:34:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741896_1074 (size=5431) 2024-11-20T22:34:59,306 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/ec39b0e9bc5a45c0a939bb0a545371e4 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/info/ec39b0e9bc5a45c0a939bb0a545371e4 2024-11-20T22:34:59,310 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:59,312 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1] 2024-11-20T22:34:59,313 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T22:34:59,314 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/info/ec39b0e9bc5a45c0a939bb0a545371e4, entries=36, sequenceid=18, filesize=10.9 K 2024-11-20T22:34:59,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T22:34:59,316 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/ns/abdd585f6c60486fbf4629ab7ecdd9c0 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/ns/abdd585f6c60486fbf4629ab7ecdd9c0 2024-11-20T22:34:59,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36989 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:45158 deadline: 1732142159316, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,317 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 9f31b4695a16,36989,1732142077539 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T22:34:59,317 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 9f31b4695a16,36989,1732142077539 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T22:34:59,317 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1 from cache 2024-11-20T22:34:59,323 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/ns/abdd585f6c60486fbf4629ab7ecdd9c0, entries=2, sequenceid=18, filesize=5.0 K 2024-11-20T22:34:59,324 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/table/2514927885324fe2906a075373c8f051 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/table/2514927885324fe2906a075373c8f051 2024-11-20T22:34:59,333 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/table/2514927885324fe2906a075373c8f051, entries=2, sequenceid=18, filesize=5.3 K 2024-11-20T22:34:59,333 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=18, compaction requested=false; wal=null 2024-11-20T22:34:59,334 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T22:34:59,336 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T22:34:59,336 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T22:34:59,337 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T22:34:59,338 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T22:34:59,340 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T22:34:59,341 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67969900, jitterRate=0.01283043622970581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T22:34:59,342 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T22:34:59,342 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732142099147Writing region info on filesystem at 1732142099147Initializing all the Stores at 1732142099148 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142099148Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142099148Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142099148Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732142099148Obtaining lock to block concurrent updates at 1732142099189 (+41 ms)Preparing flush snapshotting stores in 1588230740 at 1732142099189Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732142099189Flushing stores of hbase:meta,,1.1588230740 at 1732142099189Flushing 1588230740/info: creating writer at 1732142099189Flushing 1588230740/info: appending metadata at 1732142099213 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732142099213Flushing 1588230740/ns: creating writer at 1732142099229 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732142099248 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732142099248Flushing 1588230740/table: creating writer at 1732142099266 (+18 ms)Flushing 1588230740/table: appending metadata at 1732142099282 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732142099282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77dde095: reopening flushed file at 1732142099305 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c5a8814: reopening flushed file at 1732142099314 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1393d009: reopening flushed file at 1732142099323 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=18, compaction requested=false; wal=null at 1732142099333 (+10 ms)Cleaning up temporary data from old regions at 1732142099336 (+3 ms)Running coprocessor post-open hooks at 1732142099342 (+6 ms)Region opened successfully at 1732142099342 2024-11-20T22:34:59,344 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732142099092 2024-11-20T22:34:59,348 DEBUG [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T22:34:59,348 INFO [RS_OPEN_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T22:34:59,349 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,350 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f31b4695a16,36989,1732142077539, state=OPEN 2024-11-20T22:34:59,368 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9f31b4695a16,36989,1732142077539, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T22:34:59,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:59,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:59,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:59,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:59,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T22:34:59,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T22:34:59,411 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T22:34:59,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9f31b4695a16,36989,1732142077539 in 472 msec 2024-11-20T22:34:59,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-20T22:34:59,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2024-11-20T22:34:59,421 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 9f31b4695a16,43093,1732142077335 had 2 regions 2024-11-20T22:34:59,422 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9f31b4695a16,43093,1732142077335, splitWal=true, meta=true, isMeta: false 2024-11-20T22:34:59,424 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9f31b4695a16,43093,1732142077335 WAL count=1, meta=false 2024-11-20T22:34:59,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9f31b4695a16%2C43093%2C1732142077335.1732142079497}] 2024-11-20T22:34:59,430 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9f31b4695a16%2C43093%2C1732142077335.1732142079497, worker=9f31b4695a16,36989,1732142077539}] 2024-11-20T22:34:59,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36989 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-20T22:34:59,610 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497, size=2.2 K (2288bytes) 2024-11-20T22:34:59,610 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 2024-11-20T22:34:59,610 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 after 0ms 2024-11-20T22:34:59,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:34:59,643 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 took 34ms 2024-11-20T22:34:59,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 5e972e424facefaa32be2f83dd0a8f47: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-20T22:34:59,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 so closing down 2024-11-20T22:34:59,646 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:34:59,646 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:34:59,647 INFO [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 4 ms; skipped=6; WAL=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-20T22:34:59,647 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497, journal: Splitting hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497, size=2.2 K (2288bytes) at 1732142099610Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 so closing down at 1732142099646 (+36 ms)3 split writer threads finished at 1732142099647 (+1 ms)Processed 6 edits across 0 Regions in 4 ms; skipped=6; WAL=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732142099647 2024-11-20T22:34:59,647 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 2024-11-20T22:34:59,647 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9f31b4695a16:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-20T22:34:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-20T22:34:59,652 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting/9f31b4695a16%2C43093%2C1732142077335.1732142079497 to hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs 2024-11-20T22:34:59,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-20T22:34:59,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9f31b4695a16%2C43093%2C1732142077335.1732142079497, worker=9f31b4695a16,36989,1732142077539 in 221 msec 2024-11-20T22:34:59,659 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-20T22:34:59,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9f31b4695a16%2C43093%2C1732142077335.1732142079497, worker=9f31b4695a16,36989,1732142077539 in 235 msec 2024-11-20T22:34:59,665 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/WALs/9f31b4695a16,43093,1732142077335-splitting dir is empty, no logs to split. 2024-11-20T22:34:59,666 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9f31b4695a16,43093,1732142077335 WAL count=0, meta=false 2024-11-20T22:34:59,666 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9f31b4695a16,43093,1732142077335 WAL splitting is done? wals=0, meta=false 2024-11-20T22:34:59,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN}] 2024-11-20T22:34:59,670 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN 2024-11-20T22:34:59,671 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T22:34:59,821 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(204): Hosts are {9f31b4695a16=0} racks are {/default-rack=0} 2024-11-20T22:34:59,822 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T22:34:59,822 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T22:34:59,822 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T22:34:59,822 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T22:34:59,822 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T22:34:59,822 INFO [9f31b4695a16:40769 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T22:34:59,822 DEBUG [9f31b4695a16:40769 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T22:34:59,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPENING, regionLocation=9f31b4695a16,36989,1732142077539 2024-11-20T22:34:59,824 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9f31b4695a16:43093 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9f31b4695a16/172.17.0.2:43093 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:34:59,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1, error=java.net.ConnectException: Call to address=9f31b4695a16:43093 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9f31b4695a16/172.17.0.2:43093 2024-11-20T22:34:59,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T22:34:59,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,43093,1732142077335, seqNum=-1 from cache 2024-11-20T22:34:59,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9f31b4695a16:43093 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9f31b4695a16/172.17.0.2:43093 2024-11-20T22:34:59,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:34:59,931 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1] 2024-11-20T22:34:59,931 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T22:34:59,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44255, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T22:34:59,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN because future has completed 2024-11-20T22:34:59,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,36989,1732142077539}] 2024-11-20T22:35:00,094 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:00,094 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e972e424facefaa32be2f83dd0a8f47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:00,094 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,094 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:00,094 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,094 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,096 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,097 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf1 2024-11-20T22:35:00,097 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,105 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf1/a7ef913c2c2b46508600f57b64591dff 2024-11-20T22:35:00,105 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,105 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,106 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e972e424facefaa32be2f83dd0a8f47 columnFamilyName cf2 2024-11-20T22:35:00,106 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,114 DEBUG [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/cf2/14f0bfd33bc844e1aa72976291f33410 2024-11-20T22:35:00,114 INFO [StoreOpener-5e972e424facefaa32be2f83dd0a8f47-1 {}] regionserver.HStore(327): Store=5e972e424facefaa32be2f83dd0a8f47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,114 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,115 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,116 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,117 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,117 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,118 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T22:35:00,119 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,120 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 5e972e424facefaa32be2f83dd0a8f47; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59830047, jitterRate=-0.10846282541751862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T22:35:00,120 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:00,121 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 5e972e424facefaa32be2f83dd0a8f47: Running coprocessor pre-open hook at 1732142100094Writing region info on filesystem at 1732142100094Initializing all the Stores at 1732142100095 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100095Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100095Cleaning up temporary data from old regions at 1732142100117 (+22 ms)Running coprocessor post-open hooks at 1732142100120 (+3 ms)Region opened successfully at 1732142100121 (+1 ms) 2024-11-20T22:35:00,122 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., pid=21, masterSystemTime=1732142100090 2024-11-20T22:35:00,126 DEBUG [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:00,127 INFO [RS_OPEN_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:00,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=5e972e424facefaa32be2f83dd0a8f47, regionState=OPEN, openSeqNum=18, regionLocation=9f31b4695a16,36989,1732142077539 2024-11-20T22:35:00,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,36989,1732142077539 because future has completed 2024-11-20T22:35:00,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T22:35:00,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e972e424facefaa32be2f83dd0a8f47, server=9f31b4695a16,36989,1732142077539 in 197 msec 2024-11-20T22:35:00,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-20T22:35:00,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5e972e424facefaa32be2f83dd0a8f47, ASSIGN in 468 msec 2024-11-20T22:35:00,139 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 9f31b4695a16,43093,1732142077335 after splitting done 2024-11-20T22:35:00,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9f31b4695a16,43093,1732142077335, splitWal=true, meta=true in 1.6980 sec 2024-11-20T22:35:00,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T22:35:00,331 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f31b4695a16,36989,1732142077539, seqNum=-1] 2024-11-20T22:35:00,450 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9f31b4695a16:43093 this server is in the failed servers list 2024-11-20T22:35:00,451 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9f31b4695a16:43093 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9f31b4695a16:43093 2024-11-20T22:35:00,451 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9f31b4695a16:43093 2024-11-20T22:35:00,451 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,43093,1732142077335, seqNum=12 from cache 2024-11-20T22:35:00,668 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., hostname=9f31b4695a16,36989,1732142077539, seqNum=18] 2024-11-20T22:35:00,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 416), OpenFileDescriptor=1044 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 424) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=856 (was 1269) 2024-11-20T22:35:00,686 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1044 is superior to 1024 2024-11-20T22:35:00,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1044, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=856 2024-11-20T22:35:00,698 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1044 is superior to 1024 2024-11-20T22:35:00,720 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:00,723 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:00,724 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:35:00,726 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63386021, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-63386021, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:00,741 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63386021/hregion-63386021.1732142100727, exclude list is [], retry=0 2024-11-20T22:35:00,744 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:00,744 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:00,746 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:00,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63386021/hregion-63386021.1732142100727 2024-11-20T22:35:00,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:35:00,785 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2e16d5a67e3d317fbccefdb0184e4381, NAME => 'testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:35:00,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741898_1076 (size=67) 2024-11-20T22:35:00,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741898_1076 (size=67) 2024-11-20T22:35:00,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741898_1076 (size=67) 2024-11-20T22:35:00,806 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:00,808 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,809 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName a 2024-11-20T22:35:00,809 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,810 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,810 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,812 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName b 2024-11-20T22:35:00,812 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,813 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,813 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,815 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName c 2024-11-20T22:35:00,815 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,815 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,816 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,821 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,821 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,822 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,826 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:35:00,827 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e16d5a67e3d317fbccefdb0184e4381; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70236885, jitterRate=0.04661114513874054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:00,828 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e16d5a67e3d317fbccefdb0184e4381: Writing region info on filesystem at 1732142100806Initializing all the Stores at 1732142100807 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100807Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100807Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100807Cleaning up temporary data from old regions at 1732142100821 (+14 ms)Region opened successfully at 1732142100827 (+6 ms) 2024-11-20T22:35:00,828 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2e16d5a67e3d317fbccefdb0184e4381, disabling compactions & flushes 2024-11-20T22:35:00,828 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:00,828 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:00,828 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. after waiting 0 ms 2024-11-20T22:35:00,828 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:00,829 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:00,829 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2e16d5a67e3d317fbccefdb0184e4381: Waiting for close lock at 1732142100828Disabling compacts and flushes for region at 1732142100828Disabling writes for close at 1732142100828Writing region close event to WAL at 1732142100829 (+1 ms)Closed at 1732142100829 2024-11-20T22:35:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741897_1075 (size=175) 2024-11-20T22:35:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741897_1075 (size=175) 2024-11-20T22:35:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741897_1075 (size=175) 2024-11-20T22:35:00,837 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:00,837 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63386021:(num 1732142100727) 2024-11-20T22:35:00,838 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:00,840 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:00,853 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, exclude list is [], retry=0 2024-11-20T22:35:00,856 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:00,856 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:00,856 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:00,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 2024-11-20T22:35:00,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:00,883 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e16d5a67e3d317fbccefdb0184e4381, NAME => 'testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:00,883 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:00,884 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,884 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,885 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,886 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName a 2024-11-20T22:35:00,887 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,887 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,887 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,888 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName b 2024-11-20T22:35:00,888 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,889 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,889 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,890 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName c 2024-11-20T22:35:00,890 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:00,890 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:00,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,893 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,893 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,894 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:00,895 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:00,896 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e16d5a67e3d317fbccefdb0184e4381; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61450905, jitterRate=-0.08431015908718109}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:00,896 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e16d5a67e3d317fbccefdb0184e4381: Writing region info on filesystem at 1732142100884Initializing all the Stores at 1732142100885 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100885Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100885Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142100885Cleaning up temporary data from old regions at 1732142100893 (+8 ms)Region opened successfully at 1732142100896 (+3 ms) 2024-11-20T22:35:00,928 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2e16d5a67e3d317fbccefdb0184e4381 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T22:35:00,949 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/a/3d370dcfac1b4f0390dfa74131ed6fa5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732142100896/Put/seqid=0 2024-11-20T22:35:00,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741900_1078 (size=5958) 2024-11-20T22:35:00,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741900_1078 (size=5958) 2024-11-20T22:35:00,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741900_1078 (size=5958) 2024-11-20T22:35:00,956 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/a/3d370dcfac1b4f0390dfa74131ed6fa5 2024-11-20T22:35:00,976 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/e1a3783709fd4ce097c5f4105c77fd46 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732142100906/Put/seqid=0 2024-11-20T22:35:00,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741901_1079 (size=5958) 2024-11-20T22:35:00,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741901_1079 (size=5958) 2024-11-20T22:35:00,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741901_1079 (size=5958) 2024-11-20T22:35:00,985 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/e1a3783709fd4ce097c5f4105c77fd46 2024-11-20T22:35:01,012 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/c/21343f73b2884fe3b87237339e626fd2 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732142100916/Put/seqid=0 2024-11-20T22:35:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741902_1080 (size=5958) 2024-11-20T22:35:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741902_1080 (size=5958) 2024-11-20T22:35:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741902_1080 (size=5958) 2024-11-20T22:35:01,423 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/c/21343f73b2884fe3b87237339e626fd2 2024-11-20T22:35:01,430 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/a/3d370dcfac1b4f0390dfa74131ed6fa5 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/a/3d370dcfac1b4f0390dfa74131ed6fa5 2024-11-20T22:35:01,436 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/a/3d370dcfac1b4f0390dfa74131ed6fa5, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T22:35:01,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/e1a3783709fd4ce097c5f4105c77fd46 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/e1a3783709fd4ce097c5f4105c77fd46 2024-11-20T22:35:01,442 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/e1a3783709fd4ce097c5f4105c77fd46, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T22:35:01,443 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/c/21343f73b2884fe3b87237339e626fd2 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/c/21343f73b2884fe3b87237339e626fd2 2024-11-20T22:35:01,448 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/c/21343f73b2884fe3b87237339e626fd2, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T22:35:01,449 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2e16d5a67e3d317fbccefdb0184e4381 in 521ms, sequenceid=33, compaction requested=false 2024-11-20T22:35:01,449 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2e16d5a67e3d317fbccefdb0184e4381: 2024-11-20T22:35:01,449 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2e16d5a67e3d317fbccefdb0184e4381, disabling compactions & flushes 2024-11-20T22:35:01,449 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. after waiting 0 ms 2024-11-20T22:35:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:01,451 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. 2024-11-20T22:35:01,451 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2e16d5a67e3d317fbccefdb0184e4381: Waiting for close lock at 1732142101449Disabling compacts and flushes for region at 1732142101449Disabling writes for close at 1732142101450 (+1 ms)Writing region close event to WAL at 1732142101451 (+1 ms)Closed at 1732142101451 2024-11-20T22:35:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741899_1077 (size=6311) 2024-11-20T22:35:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741899_1077 (size=6311) 2024-11-20T22:35:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741899_1077 (size=6311) 2024-11-20T22:35:01,458 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/e1a3783709fd4ce097c5f4105c77fd46 to hdfs://localhost:46165/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/e1a3783709fd4ce097c5f4105c77fd46 2024-11-20T22:35:01,475 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, size=6.2 K (6311bytes) 2024-11-20T22:35:01,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 2024-11-20T22:35:01,476 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 after 0ms 2024-11-20T22:35:01,504 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:01,505 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 took 29ms 2024-11-20T22:35:01,507 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 so closing down 2024-11-20T22:35:01,507 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:01,508 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732142100840.temp 2024-11-20T22:35:01,537 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp 2024-11-20T22:35:01,538 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741903_1081 (size=5575) 2024-11-20T22:35:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741903_1081 (size=5575) 2024-11-20T22:35:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741903_1081 (size=5575) 2024-11-20T22:35:01,546 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:01,548 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 2024-11-20T22:35:01,548 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-20T22:35:01,548 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, size=6.2 K (6311bytes) at 1732142101475Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 so closing down at 1732142101507 (+32 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp at 1732142101537 (+30 ms)3 split writer threads finished at 1732142101538 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732142101546 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000003-wal.1732142100840.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 at 1732142101548 (+2 ms)Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732142101548 2024-11-20T22:35:01,550 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142100840 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142100840 2024-11-20T22:35:01,551 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 2024-11-20T22:35:01,551 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:01,553 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:01,569 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142101553, exclude list is [], retry=0 2024-11-20T22:35:01,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:01,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:01,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:01,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142101553 2024-11-20T22:35:01,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:01,601 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e16d5a67e3d317fbccefdb0184e4381, NAME => 'testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:01,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:01,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,601 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,603 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,603 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName a 2024-11-20T22:35:01,604 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,610 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/a/3d370dcfac1b4f0390dfa74131ed6fa5 2024-11-20T22:35:01,610 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,610 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,612 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName b 2024-11-20T22:35:01,612 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,612 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,612 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,613 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e16d5a67e3d317fbccefdb0184e4381 columnFamilyName c 2024-11-20T22:35:01,613 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,619 DEBUG [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/c/21343f73b2884fe3b87237339e626fd2 2024-11-20T22:35:01,619 INFO [StoreOpener-2e16d5a67e3d317fbccefdb0184e4381-1 {}] regionserver.HStore(327): Store=2e16d5a67e3d317fbccefdb0184e4381/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,619 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,620 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,622 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,623 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 2024-11-20T22:35:01,651 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:01,653 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 2024-11-20T22:35:01,653 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2e16d5a67e3d317fbccefdb0184e4381 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T22:35:01,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/822b48d13b0b4c8d9ec3b7feaadffc3c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732142100906/Put/seqid=0 2024-11-20T22:35:01,675 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T22:35:01,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741905_1083 (size=5958) 2024-11-20T22:35:01,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741905_1083 (size=5958) 2024-11-20T22:35:01,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741905_1083 (size=5958) 2024-11-20T22:35:01,685 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/822b48d13b0b4c8d9ec3b7feaadffc3c 2024-11-20T22:35:01,711 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/.tmp/b/822b48d13b0b4c8d9ec3b7feaadffc3c as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/822b48d13b0b4c8d9ec3b7feaadffc3c 2024-11-20T22:35:01,739 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/b/822b48d13b0b4c8d9ec3b7feaadffc3c, entries=10, sequenceid=32, filesize=5.8 K 2024-11-20T22:35:01,740 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2e16d5a67e3d317fbccefdb0184e4381 in 87ms, sequenceid=32, compaction requested=false; wal=null 2024-11-20T22:35:01,741 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/0000000000000000032 2024-11-20T22:35:01,742 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,743 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,744 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:01,746 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e16d5a67e3d317fbccefdb0184e4381 2024-11-20T22:35:01,748 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/2e16d5a67e3d317fbccefdb0184e4381/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-20T22:35:01,749 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e16d5a67e3d317fbccefdb0184e4381; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63513372, jitterRate=-0.0535770058631897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:01,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e16d5a67e3d317fbccefdb0184e4381: Writing region info on filesystem at 1732142101601Initializing all the Stores at 1732142101602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101602Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101602Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101602Obtaining lock to block concurrent updates at 1732142101653 (+51 ms)Preparing flush snapshotting stores in 2e16d5a67e3d317fbccefdb0184e4381 at 1732142101653Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732142101653Flushing stores of testReplayEditsWrittenViaHRegion,,1732142100721.2e16d5a67e3d317fbccefdb0184e4381. at 1732142101653Flushing 2e16d5a67e3d317fbccefdb0184e4381/b: creating writer at 1732142101654 (+1 ms)Flushing 2e16d5a67e3d317fbccefdb0184e4381/b: appending metadata at 1732142101667 (+13 ms)Flushing 2e16d5a67e3d317fbccefdb0184e4381/b: closing flushed file at 1732142101667Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be66f09: reopening flushed file at 1732142101694 (+27 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2e16d5a67e3d317fbccefdb0184e4381 in 87ms, sequenceid=32, compaction requested=false; wal=null at 1732142101740 (+46 ms)Cleaning up temporary data from old regions at 1732142101743 (+3 ms)Region opened successfully at 1732142101750 (+7 ms) 2024-11-20T22:35:01,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:56502 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:38466 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:35954 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:38268 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:56356 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:35770 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1044) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 454), ProcessCount=11 (was 11), AvailableMemoryMB=818 (was 856) 2024-11-20T22:35:01,771 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-20T22:35:01,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=818 2024-11-20T22:35:01,783 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-20T22:35:01,797 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:01,799 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:01,800 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:35:01,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02596598, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-02596598, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:01,817 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02596598/hregion-02596598.1732142101803, exclude list is [], retry=0 2024-11-20T22:35:01,819 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:01,820 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:01,820 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:01,846 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02596598/hregion-02596598.1732142101803 2024-11-20T22:35:01,847 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:35:01,847 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 190ab8ec98fbed26325c3e053acc4762, NAME => 'testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:35:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741907_1085 (size=68) 2024-11-20T22:35:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741907_1085 (size=68) 2024-11-20T22:35:01,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741907_1085 (size=68) 2024-11-20T22:35:01,856 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:01,857 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,859 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName a 2024-11-20T22:35:01,859 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,860 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,860 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,861 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName b 2024-11-20T22:35:01,861 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,862 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,862 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,864 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName c 2024-11-20T22:35:01,864 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:01,864 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:01,864 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,865 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,866 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,867 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,867 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,867 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:01,869 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:01,871 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:35:01,872 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 190ab8ec98fbed26325c3e053acc4762; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61649778, jitterRate=-0.08134672045707703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:01,872 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 190ab8ec98fbed26325c3e053acc4762: Writing region info on filesystem at 1732142101856Initializing all the Stores at 1732142101857 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101857Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101857Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142101857Cleaning up temporary data from old regions at 1732142101867 (+10 ms)Region opened successfully at 1732142101872 (+5 ms) 2024-11-20T22:35:01,873 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 190ab8ec98fbed26325c3e053acc4762, disabling compactions & flushes 2024-11-20T22:35:01,873 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:01,873 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:01,873 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. after waiting 0 ms 2024-11-20T22:35:01,873 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:01,873 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:01,873 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 190ab8ec98fbed26325c3e053acc4762: Waiting for close lock at 1732142101873Disabling compacts and flushes for region at 1732142101873Disabling writes for close at 1732142101873Writing region close event to WAL at 1732142101873Closed at 1732142101873 2024-11-20T22:35:01,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741906_1084 (size=175) 2024-11-20T22:35:01,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741906_1084 (size=175) 2024-11-20T22:35:01,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741906_1084 (size=175) 2024-11-20T22:35:01,878 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:01,878 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02596598:(num 1732142101803) 2024-11-20T22:35:01,878 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:01,880 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:01,894 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, exclude list is [], retry=0 2024-11-20T22:35:01,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:01,898 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:01,898 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:01,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 2024-11-20T22:35:01,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:02,003 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 190ab8ec98fbed26325c3e053acc4762, NAME => 'testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:02,005 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,005 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:02,006 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,006 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,008 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,010 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName a 2024-11-20T22:35:02,010 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:02,011 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:02,011 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,012 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName b 2024-11-20T22:35:02,012 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:02,013 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:02,013 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,014 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName c 2024-11-20T22:35:02,014 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:02,015 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:02,015 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,016 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,018 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,018 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,019 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:02,021 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,022 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 190ab8ec98fbed26325c3e053acc4762; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69964962, jitterRate=0.0425591766834259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:02,022 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:02,023 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 190ab8ec98fbed26325c3e053acc4762: Running coprocessor pre-open hook at 1732142102006Writing region info on filesystem at 1732142102006Initializing all the Stores at 1732142102007 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142102007Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142102007Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142102007Cleaning up temporary data from old regions at 1732142102018 (+11 ms)Running coprocessor post-open hooks at 1732142102022 (+4 ms)Region opened successfully at 1732142102023 (+1 ms) 2024-11-20T22:35:02,044 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 190ab8ec98fbed26325c3e053acc4762 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-20T22:35:02,045 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:03,046 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:03,618 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T22:35:04,047 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:05,048 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:05,614 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T22:35:06,048 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:06,849 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-20T22:35:06,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-20T22:35:06,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T22:35:06,851 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-20T22:35:06,852 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-20T22:35:06,852 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-20T22:35:07,049 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:07,509 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T22:35:08,051 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:09,052 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:10,052 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:11,053 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 190ab8ec98fbed26325c3e053acc4762/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:11,055 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 190ab8ec98fbed26325c3e053acc4762: 2024-11-20T22:35:11,055 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,072 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 190ab8ec98fbed26325c3e053acc4762: 2024-11-20T22:35:11,072 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-20T22:35:11,072 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 190ab8ec98fbed26325c3e053acc4762, disabling compactions & flushes 2024-11-20T22:35:11,072 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,072 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,072 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. after waiting 0 ms 2024-11-20T22:35:11,072 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,073 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. 2024-11-20T22:35:11,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 190ab8ec98fbed26325c3e053acc4762: Waiting for close lock at 1732142111072Running coprocessor pre-close hooks at 1732142111072Disabling compacts and flushes for region at 1732142111072Disabling writes for close at 1732142111072Writing region close event to WAL at 1732142111073 (+1 ms)Running coprocessor post-close hooks at 1732142111073Closed at 1732142111073 2024-11-20T22:35:11,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741908_1086 (size=4236) 2024-11-20T22:35:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741908_1086 (size=4236) 2024-11-20T22:35:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741908_1086 (size=4236) 2024-11-20T22:35:11,113 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, size=4.1 K (4236bytes) 2024-11-20T22:35:11,113 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 2024-11-20T22:35:11,114 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 after 1ms 2024-11-20T22:35:11,138 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:11,138 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 took 25ms 2024-11-20T22:35:11,141 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 so closing down 2024-11-20T22:35:11,141 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:11,142 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732142101880.temp 2024-11-20T22:35:11,176 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp 2024-11-20T22:35:11,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:11,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741909_1087 (size=3245) 2024-11-20T22:35:11,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741909_1087 (size=3245) 2024-11-20T22:35:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741909_1087 (size=3245) 2024-11-20T22:35:11,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:11,186 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 2024-11-20T22:35:11,187 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 47 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-20T22:35:11,187 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, size=4.1 K (4236bytes) at 1732142111113Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 so closing down at 1732142111141 (+28 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp at 1732142111176 (+35 ms)3 split writer threads finished at 1732142111177 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732142111185 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000004-wal.1732142101880.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 at 1732142111186 (+1 ms)Processed 23 edits across 1 Regions in 47 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732142111187 (+1 ms) 2024-11-20T22:35:11,188 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142101880 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142101880 2024-11-20T22:35:11,189 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 2024-11-20T22:35:11,189 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:11,191 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:11,203 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142111191, exclude list is [], retry=0 2024-11-20T22:35:11,206 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:11,206 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:11,207 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:11,231 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142111191 2024-11-20T22:35:11,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:35:11,232 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 190ab8ec98fbed26325c3e053acc4762, NAME => 'testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:11,232 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,232 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:11,233 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,233 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,235 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,235 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName a 2024-11-20T22:35:11,235 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:11,236 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:11,236 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,237 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName b 2024-11-20T22:35:11,237 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:11,237 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:11,237 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,238 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 190ab8ec98fbed26325c3e053acc4762 columnFamilyName c 2024-11-20T22:35:11,238 DEBUG [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:11,238 INFO [StoreOpener-190ab8ec98fbed26325c3e053acc4762-1 {}] regionserver.HStore(327): Store=190ab8ec98fbed26325c3e053acc4762/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:11,239 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,241 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,241 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 2024-11-20T22:35:11,269 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:11,271 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 2024-11-20T22:35:11,271 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 190ab8ec98fbed26325c3e053acc4762 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-20T22:35:11,286 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/a/7995cd8e9ed848b49c48368a1d213ef0 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732142111062/Put/seqid=0 2024-11-20T22:35:11,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741911_1089 (size=5523) 2024-11-20T22:35:11,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741911_1089 (size=5523) 2024-11-20T22:35:11,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741911_1089 (size=5523) 2024-11-20T22:35:11,293 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/a/7995cd8e9ed848b49c48368a1d213ef0 2024-11-20T22:35:11,313 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/b/23c0b43f2c5b4a92800d6b42115eca6e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732142111056/Put/seqid=0 2024-11-20T22:35:11,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741912_1090 (size=5524) 2024-11-20T22:35:11,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741912_1090 (size=5524) 2024-11-20T22:35:11,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741912_1090 (size=5524) 2024-11-20T22:35:11,321 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/b/23c0b43f2c5b4a92800d6b42115eca6e 2024-11-20T22:35:11,346 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/c/38ca7a95e19542cf9bf730c01cac0b46 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732142111059/Put/seqid=0 2024-11-20T22:35:11,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741913_1091 (size=5457) 2024-11-20T22:35:11,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741913_1091 (size=5457) 2024-11-20T22:35:11,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741913_1091 (size=5457) 2024-11-20T22:35:11,358 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/c/38ca7a95e19542cf9bf730c01cac0b46 2024-11-20T22:35:11,364 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/a/7995cd8e9ed848b49c48368a1d213ef0 as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/a/7995cd8e9ed848b49c48368a1d213ef0 2024-11-20T22:35:11,371 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/a/7995cd8e9ed848b49c48368a1d213ef0, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T22:35:11,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/b/23c0b43f2c5b4a92800d6b42115eca6e as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/b/23c0b43f2c5b4a92800d6b42115eca6e 2024-11-20T22:35:11,379 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/b/23c0b43f2c5b4a92800d6b42115eca6e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T22:35:11,380 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/.tmp/c/38ca7a95e19542cf9bf730c01cac0b46 as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/c/38ca7a95e19542cf9bf730c01cac0b46 2024-11-20T22:35:11,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/c/38ca7a95e19542cf9bf730c01cac0b46, entries=6, sequenceid=26, filesize=5.3 K 2024-11-20T22:35:11,388 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 190ab8ec98fbed26325c3e053acc4762 in 117ms, sequenceid=26, compaction requested=false; wal=null 2024-11-20T22:35:11,389 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/0000000000000000026 2024-11-20T22:35:11,390 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,390 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,391 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:11,393 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,396 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/190ab8ec98fbed26325c3e053acc4762/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-20T22:35:11,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 190ab8ec98fbed26325c3e053acc4762; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62316340, jitterRate=-0.07141417264938354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:11,397 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 190ab8ec98fbed26325c3e053acc4762 2024-11-20T22:35:11,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 190ab8ec98fbed26325c3e053acc4762: Running coprocessor pre-open hook at 1732142111233Writing region info on filesystem at 1732142111233Initializing all the Stores at 1732142111234 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142111234Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142111234Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142111234Obtaining lock to block concurrent updates at 1732142111271 (+37 ms)Preparing flush snapshotting stores in 190ab8ec98fbed26325c3e053acc4762 at 1732142111271Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732142111272 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732142101798.190ab8ec98fbed26325c3e053acc4762. at 1732142111272Flushing 190ab8ec98fbed26325c3e053acc4762/a: creating writer at 1732142111272Flushing 190ab8ec98fbed26325c3e053acc4762/a: appending metadata at 1732142111285 (+13 ms)Flushing 190ab8ec98fbed26325c3e053acc4762/a: closing flushed file at 1732142111285Flushing 190ab8ec98fbed26325c3e053acc4762/b: creating writer at 1732142111298 (+13 ms)Flushing 190ab8ec98fbed26325c3e053acc4762/b: appending metadata at 1732142111312 (+14 ms)Flushing 190ab8ec98fbed26325c3e053acc4762/b: closing flushed file at 1732142111312Flushing 190ab8ec98fbed26325c3e053acc4762/c: creating writer at 1732142111326 (+14 ms)Flushing 190ab8ec98fbed26325c3e053acc4762/c: appending metadata at 1732142111345 (+19 ms)Flushing 190ab8ec98fbed26325c3e053acc4762/c: closing flushed file at 1732142111345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@673ee18b: reopening flushed file at 1732142111363 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5287e27e: reopening flushed file at 1732142111371 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@583e20fe: reopening flushed file at 1732142111379 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 190ab8ec98fbed26325c3e053acc4762 in 117ms, sequenceid=26, compaction requested=false; wal=null at 1732142111388 (+9 ms)Cleaning up temporary data from old regions at 1732142111391 (+3 ms)Running coprocessor post-open hooks at 1732142111397 (+6 ms)Region opened successfully at 1732142111398 (+1 ms) 2024-11-20T22:35:11,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=426 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46402 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46028 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46406 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:40674 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:45996 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:40662 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 425), ProcessCount=11 (was 11), AvailableMemoryMB=759 (was 818) 2024-11-20T22:35:11,423 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T22:35:11,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=426, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=755 2024-11-20T22:35:11,438 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T22:35:11,459 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:11,462 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:11,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:35:11,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68551376, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-68551376, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:11,487 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68551376/hregion-68551376.1732142111469, exclude list is [], retry=0 2024-11-20T22:35:11,491 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:11,491 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:11,492 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:11,522 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68551376/hregion-68551376.1732142111469 2024-11-20T22:35:11,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:11,523 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 158aa2eefb8c8b94ee9d972278b0e1e8, NAME => 'testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:35:11,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741915_1093 (size=61) 2024-11-20T22:35:11,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741915_1093 (size=61) 2024-11-20T22:35:11,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741915_1093 (size=61) 2024-11-20T22:35:11,533 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:11,534 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,536 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 158aa2eefb8c8b94ee9d972278b0e1e8 columnFamilyName a 2024-11-20T22:35:11,536 DEBUG [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:11,537 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(327): Store=158aa2eefb8c8b94ee9d972278b0e1e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:11,537 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,538 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,539 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,539 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,541 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,545 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:35:11,545 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 158aa2eefb8c8b94ee9d972278b0e1e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60141735, jitterRate=-0.10381831228733063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:35:11,546 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Writing region info on filesystem at 1732142111533Initializing all the Stores at 1732142111534 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142111534Cleaning up temporary data from old regions at 1732142111539 (+5 ms)Region opened successfully at 1732142111546 (+7 ms) 2024-11-20T22:35:11,546 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 158aa2eefb8c8b94ee9d972278b0e1e8, disabling compactions & flushes 2024-11-20T22:35:11,546 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,546 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,546 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. after waiting 0 ms 2024-11-20T22:35:11,546 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,546 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,547 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Waiting for close lock at 1732142111546Disabling compacts and flushes for region at 1732142111546Disabling writes for close at 1732142111546Writing region close event to WAL at 1732142111546Closed at 1732142111546 2024-11-20T22:35:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741914_1092 (size=175) 2024-11-20T22:35:11,550 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-68551376/hregion-68551376.1732142111469 not finished, retry = 0 2024-11-20T22:35:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741914_1092 (size=175) 2024-11-20T22:35:11,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741914_1092 (size=175) 2024-11-20T22:35:11,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:11,653 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68551376:(num 1732142111469) 2024-11-20T22:35:11,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:11,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:11,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, exclude list is [], retry=0 2024-11-20T22:35:11,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:11,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:11,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:11,697 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 2024-11-20T22:35:11,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:11,698 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 158aa2eefb8c8b94ee9d972278b0e1e8, NAME => 'testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:11,698 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:11,698 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,698 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,700 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,701 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 158aa2eefb8c8b94ee9d972278b0e1e8 columnFamilyName a 2024-11-20T22:35:11,701 DEBUG [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:11,701 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(327): Store=158aa2eefb8c8b94ee9d972278b0e1e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:11,701 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,702 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,704 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,704 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,705 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:11,706 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 158aa2eefb8c8b94ee9d972278b0e1e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69338398, jitterRate=0.03322264552116394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:35:11,706 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Writing region info on filesystem at 1732142111698Initializing all the Stores at 1732142111699 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142111699Cleaning up temporary data from old regions at 1732142111704 (+5 ms)Region opened successfully at 1732142111706 (+2 ms) 2024-11-20T22:35:11,716 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 158aa2eefb8c8b94ee9d972278b0e1e8, disabling compactions & flushes 2024-11-20T22:35:11,716 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,716 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,716 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. after waiting 0 ms 2024-11-20T22:35:11,716 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,717 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,717 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. 2024-11-20T22:35:11,717 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Waiting for close lock at 1732142111716Disabling compacts and flushes for region at 1732142111716Disabling writes for close at 1732142111716Writing region close event to WAL at 1732142111717 (+1 ms)Closed at 1732142111717 2024-11-20T22:35:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741916_1094 (size=1795) 2024-11-20T22:35:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741916_1094 (size=1795) 2024-11-20T22:35:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741916_1094 (size=1795) 2024-11-20T22:35:11,736 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, size=1.8 K (1795bytes) 2024-11-20T22:35:11,736 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 2024-11-20T22:35:11,736 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 after 0ms 2024-11-20T22:35:11,764 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:11,764 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 took 28ms 2024-11-20T22:35:11,766 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 so closing down 2024-11-20T22:35:11,766 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:11,767 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732142111655.temp 2024-11-20T22:35:11,794 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp 2024-11-20T22:35:11,794 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:11,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741917_1095 (size=1795) 2024-11-20T22:35:11,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741917_1095 (size=1795) 2024-11-20T22:35:11,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741917_1095 (size=1795) 2024-11-20T22:35:11,801 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:11,802 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp to hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 2024-11-20T22:35:11,802 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-20T22:35:11,803 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, journal: Splitting hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, size=1.8 K (1795bytes) at 1732142111736Finishing writing output for hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 so closing down at 1732142111766 (+30 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp at 1732142111794 (+28 ms)3 split writer threads finished at 1732142111794Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732142111801 (+7 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000003-wal.1732142111655.temp to hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 at 1732142111802 (+1 ms)Processed 10 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732142111802 2024-11-20T22:35:11,804 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142111655 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142111655 2024-11-20T22:35:11,805 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 2024-11-20T22:35:11,833 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:12,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:12,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:12,208 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142112196, exclude list is [], retry=0 2024-11-20T22:35:12,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:12,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:12,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:12,235 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142112196 2024-11-20T22:35:12,236 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:35:12,236 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 158aa2eefb8c8b94ee9d972278b0e1e8, NAME => 'testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:12,236 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:12,236 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,236 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,239 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,241 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 158aa2eefb8c8b94ee9d972278b0e1e8 columnFamilyName a 2024-11-20T22:35:12,241 DEBUG [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,242 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(327): Store=158aa2eefb8c8b94ee9d972278b0e1e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,242 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,245 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 2024-11-20T22:35:12,270 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:12,271 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 2024-11-20T22:35:12,272 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 158aa2eefb8c8b94ee9d972278b0e1e8 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-20T22:35:12,338 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/.tmp/a/a0aa970e4b5a42139e9dcba7f377ee6a is 79, key is testDatalossWhenInputError/a:x0/1732142111707/Put/seqid=0 2024-11-20T22:35:12,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741919_1097 (size=5808) 2024-11-20T22:35:12,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741919_1097 (size=5808) 2024-11-20T22:35:12,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741919_1097 (size=5808) 2024-11-20T22:35:12,345 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/.tmp/a/a0aa970e4b5a42139e9dcba7f377ee6a 2024-11-20T22:35:12,355 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/.tmp/a/a0aa970e4b5a42139e9dcba7f377ee6a as hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/a/a0aa970e4b5a42139e9dcba7f377ee6a 2024-11-20T22:35:12,365 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/a/a0aa970e4b5a42139e9dcba7f377ee6a, entries=10, sequenceid=12, filesize=5.7 K 2024-11-20T22:35:12,365 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 158aa2eefb8c8b94ee9d972278b0e1e8 in 94ms, sequenceid=12, compaction requested=false; wal=null 2024-11-20T22:35:12,366 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/0000000000000000012 2024-11-20T22:35:12,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,371 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,374 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-20T22:35:12,375 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 158aa2eefb8c8b94ee9d972278b0e1e8; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63236499, jitterRate=-0.05770273506641388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:35:12,376 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Writing region info on filesystem at 1732142112236Initializing all the Stores at 1732142112239 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112239Obtaining lock to block concurrent updates at 1732142112272 (+33 ms)Preparing flush snapshotting stores in 158aa2eefb8c8b94ee9d972278b0e1e8 at 1732142112272Finished memstore snapshotting testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732142112272Flushing stores of testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8. at 1732142112272Flushing 158aa2eefb8c8b94ee9d972278b0e1e8/a: creating writer at 1732142112272Flushing 158aa2eefb8c8b94ee9d972278b0e1e8/a: appending metadata at 1732142112337 (+65 ms)Flushing 158aa2eefb8c8b94ee9d972278b0e1e8/a: closing flushed file at 1732142112337Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a878f06: reopening flushed file at 1732142112353 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 158aa2eefb8c8b94ee9d972278b0e1e8 in 94ms, sequenceid=12, compaction requested=false; wal=null at 1732142112365 (+12 ms)Cleaning up temporary data from old regions at 1732142112367 (+2 ms)Region opened successfully at 1732142112376 (+9 ms) 2024-11-20T22:35:12,381 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 158aa2eefb8c8b94ee9d972278b0e1e8, NAME => 'testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:12,381 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732142111460.158aa2eefb8c8b94ee9d972278b0e1e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:12,381 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,381 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,385 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,387 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 158aa2eefb8c8b94ee9d972278b0e1e8 columnFamilyName a 2024-11-20T22:35:12,387 DEBUG [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,394 DEBUG [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/a/a0aa970e4b5a42139e9dcba7f377ee6a 2024-11-20T22:35:12,394 INFO [StoreOpener-158aa2eefb8c8b94ee9d972278b0e1e8-1 {}] regionserver.HStore(327): Store=158aa2eefb8c8b94ee9d972278b0e1e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,394 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,397 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,398 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,398 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,404 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 158aa2eefb8c8b94ee9d972278b0e1e8 2024-11-20T22:35:12,407 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/158aa2eefb8c8b94ee9d972278b0e1e8/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-20T22:35:12,408 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 158aa2eefb8c8b94ee9d972278b0e1e8; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64470972, jitterRate=-0.03930765390396118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T22:35:12,409 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 158aa2eefb8c8b94ee9d972278b0e1e8: Writing region info on filesystem at 1732142112381Initializing all the Stores at 1732142112384 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112384Cleaning up temporary data from old regions at 1732142112398 (+14 ms)Region opened successfully at 1732142112409 (+11 ms) 2024-11-20T22:35:12,428 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=436 (was 426) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46028 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:36034 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46406 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:40662 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:47830 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46646 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 399) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=733 (was 755) 2024-11-20T22:35:12,428 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T22:35:12,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=436, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=732 2024-11-20T22:35:12,441 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T22:35:12,457 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:12,459 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:12,459 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:35:12,461 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78043189, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-78043189, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:12,475 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78043189/hregion-78043189.1732142112462, exclude list is [], retry=0 2024-11-20T22:35:12,478 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:12,478 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:12,479 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:12,507 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78043189/hregion-78043189.1732142112462 2024-11-20T22:35:12,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:12,508 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b5fed784dcd7a622aa63b0e259d51d65, NAME => 'testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:35:12,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741921_1099 (size=63) 2024-11-20T22:35:12,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741921_1099 (size=63) 2024-11-20T22:35:12,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741921_1099 (size=63) 2024-11-20T22:35:12,517 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:12,518 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,520 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName a 2024-11-20T22:35:12,520 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,520 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,520 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,522 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName b 2024-11-20T22:35:12,522 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,522 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,522 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,524 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName c 2024-11-20T22:35:12,524 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,524 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,524 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,525 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,525 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,527 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:12,528 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,530 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:35:12,531 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b5fed784dcd7a622aa63b0e259d51d65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58957214, jitterRate=-0.12146905064582825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:12,531 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b5fed784dcd7a622aa63b0e259d51d65: Writing region info on filesystem at 1732142112517Initializing all the Stores at 1732142112518 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112518Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112518Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112518Cleaning up temporary data from old regions at 1732142112526 (+8 ms)Region opened successfully at 1732142112531 (+5 ms) 2024-11-20T22:35:12,531 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b5fed784dcd7a622aa63b0e259d51d65, disabling compactions & flushes 2024-11-20T22:35:12,531 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:12,531 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:12,531 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. after waiting 0 ms 2024-11-20T22:35:12,531 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:12,532 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:12,532 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b5fed784dcd7a622aa63b0e259d51d65: Waiting for close lock at 1732142112531Disabling compacts and flushes for region at 1732142112531Disabling writes for close at 1732142112531Writing region close event to WAL at 1732142112532 (+1 ms)Closed at 1732142112532 2024-11-20T22:35:12,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741920_1098 (size=175) 2024-11-20T22:35:12,541 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-78043189/hregion-78043189.1732142112462 not finished, retry = 0 2024-11-20T22:35:12,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741920_1098 (size=175) 2024-11-20T22:35:12,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741920_1098 (size=175) 2024-11-20T22:35:12,643 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:12,643 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78043189:(num 1732142112462) 2024-11-20T22:35:12,643 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:12,645 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:12,658 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, exclude list is [], retry=0 2024-11-20T22:35:12,661 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:12,661 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:12,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:12,685 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 2024-11-20T22:35:12,686 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:35:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b5fed784dcd7a622aa63b0e259d51d65, NAME => 'testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,686 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,688 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,689 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName a 2024-11-20T22:35:12,689 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,690 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,690 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,691 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName b 2024-11-20T22:35:12,691 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,691 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,691 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,692 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName c 2024-11-20T22:35:12,692 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:12,692 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:12,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,695 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,695 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,695 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:12,697 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:12,698 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b5fed784dcd7a622aa63b0e259d51d65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73820678, jitterRate=0.10001382231712341}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:12,698 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b5fed784dcd7a622aa63b0e259d51d65: Writing region info on filesystem at 1732142112686Initializing all the Stores at 1732142112687 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112687Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112688 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142112688Cleaning up temporary data from old regions at 1732142112695 (+7 ms)Region opened successfully at 1732142112698 (+3 ms) 2024-11-20T22:35:12,702 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732142112702/Put/seqid=0 2024-11-20T22:35:12,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741923_1101 (size=4875) 2024-11-20T22:35:12,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741923_1101 (size=4875) 2024-11-20T22:35:12,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741923_1101 (size=4875) 2024-11-20T22:35:12,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732142112713/Put/seqid=0 2024-11-20T22:35:12,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741924_1102 (size=4875) 2024-11-20T22:35:12,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741924_1102 (size=4875) 2024-11-20T22:35:12,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741924_1102 (size=4875) 2024-11-20T22:35:12,727 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732142112727/Put/seqid=0 2024-11-20T22:35:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741925_1103 (size=4875) 2024-11-20T22:35:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741925_1103 (size=4875) 2024-11-20T22:35:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741925_1103 (size=4875) 2024-11-20T22:35:12,735 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,738 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-20T22:35:12,738 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T22:35:12,738 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,743 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-20T22:35:12,743 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T22:35:12,743 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,747 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-20T22:35:12,747 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T22:35:12,747 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b5fed784dcd7a622aa63b0e259d51d65 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-20T22:35:12,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 is 55, key is testCompactedBulkLoadedFiles/a:a/1732142112699/Put/seqid=0 2024-11-20T22:35:12,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741926_1104 (size=5107) 2024-11-20T22:35:12,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741926_1104 (size=5107) 2024-11-20T22:35:12,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741926_1104 (size=5107) 2024-11-20T22:35:12,770 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 2024-11-20T22:35:12,776 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 2024-11-20T22:35:12,782 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0, entries=1, sequenceid=4, filesize=5.0 K 2024-11-20T22:35:12,783 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for b5fed784dcd7a622aa63b0e259d51d65 in 36ms, sequenceid=4, compaction requested=false 2024-11-20T22:35:12,783 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b5fed784dcd7a622aa63b0e259d51d65: 2024-11-20T22:35:12,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ 2024-11-20T22:35:12,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ 2024-11-20T22:35:12,787 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ 2024-11-20T22:35:12,787 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 into b5fed784dcd7a622aa63b0e259d51d65/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ - updating store file list. 2024-11-20T22:35:12,792 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:12,792 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ into b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,792 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 into b5fed784dcd7a622aa63b0e259d51d65/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_) 2024-11-20T22:35:12,793 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 into b5fed784dcd7a622aa63b0e259d51d65/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ - updating store file list. 2024-11-20T22:35:12,798 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:12,798 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ into b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,798 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 into b5fed784dcd7a622aa63b0e259d51d65/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_) 2024-11-20T22:35:12,800 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 into b5fed784dcd7a622aa63b0e259d51d65/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ - updating store file list. 2024-11-20T22:35:12,806 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ff66323963734972b096084abbfd1a0c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:12,806 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ into b5fed784dcd7a622aa63b0e259d51d65/a 2024-11-20T22:35:12,806 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 into b5fed784dcd7a622aa63b0e259d51d65/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_) 2024-11-20T22:35:12,815 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T22:35:12,815 DEBUG [Time-limited test {}] regionserver.HStore(1541): b5fed784dcd7a622aa63b0e259d51d65/a is initiating major compaction (all files) 2024-11-20T22:35:12,815 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b5fed784dcd7a622aa63b0e259d51d65/a in testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:12,816 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_] into tmpdir=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp, totalSize=19.3 K 2024-11-20T22:35:12,817 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7ed05e6088ab4f4ca9f57d0b522e4cb0, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732142112699 2024-11-20T22:35:12,819 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T22:35:12,819 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T22:35:12,820 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ff66323963734972b096084abbfd1a0c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T22:35:12,843 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp/a/7466fe3b57c340d4a27a73f4e11ce69d is 55, key is testCompactedBulkLoadedFiles/a:a/1732142112699/Put/seqid=0 2024-11-20T22:35:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741927_1105 (size=6154) 2024-11-20T22:35:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741927_1105 (size=6154) 2024-11-20T22:35:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741927_1105 (size=6154) 2024-11-20T22:35:12,859 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/.tmp/a/7466fe3b57c340d4a27a73f4e11ce69d as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7466fe3b57c340d4a27a73f4e11ce69d 2024-11-20T22:35:12,865 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in b5fed784dcd7a622aa63b0e259d51d65/a of b5fed784dcd7a622aa63b0e259d51d65 into 7466fe3b57c340d4a27a73f4e11ce69d(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T22:35:12,866 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b5fed784dcd7a622aa63b0e259d51d65: 2024-11-20T22:35:12,866 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T22:35:12,866 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T22:35:12,898 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, size=0 (0bytes) 2024-11-20T22:35:12,898 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 might be still open, length is 0 2024-11-20T22:35:12,898 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 2024-11-20T22:35:12,898 WARN [IPC Server handler 1 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-20T22:35:12,899 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 after 0ms 2024-11-20T22:35:14,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:36072 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37697:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36072 dst: /127.0.0.1:37697 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37697 remote=/127.0.0.1:36072]. Total timeout mills is 60000, 58345 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:14,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:47868 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47868 dst: /127.0.0.1:36643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:14,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46670 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:45551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46670 dst: /127.0.0.1:45551 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:14,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741922_1106 (size=1794) 2024-11-20T22:35:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741922_1106 (size=1794) 2024-11-20T22:35:14,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741922_1106 (size=1794) 2024-11-20T22:35:16,575 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T22:35:16,850 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-20T22:35:16,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-20T22:35:16,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-20T22:35:16,851 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-20T22:35:16,899 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 after 4001ms 2024-11-20T22:35:16,940 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:16,940 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 took 4042ms 2024-11-20T22:35:16,942 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646; continuing. 2024-11-20T22:35:16,942 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 so closing down 2024-11-20T22:35:16,942 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:16,944 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732142112646.temp 2024-11-20T22:35:17,003 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp 2024-11-20T22:35:17,003 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741928_1107 (size=800) 2024-11-20T22:35:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741928_1107 (size=800) 2024-11-20T22:35:17,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741928_1107 (size=800) 2024-11-20T22:35:17,011 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:17,013 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp to hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 2024-11-20T22:35:17,013 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 72 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T22:35:17,013 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, journal: Splitting hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, size=0 (0bytes) at 1732142112898Finishing writing output for hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 so closing down at 1732142116942 (+4044 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp at 1732142117003 (+61 ms)3 split writer threads finished at 1732142117003Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732142117011 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000003-wal.1732142112646.temp to hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 at 1732142117013 (+2 ms)Processed 5 edits across 1 Regions in 72 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646, size=0, length=0, corrupted=false, cancelled=false at 1732142117013 2024-11-20T22:35:17,014 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142112646 2024-11-20T22:35:17,015 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 2024-11-20T22:35:17,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:17,017 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:17,030 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142117017, exclude list is [], retry=0 2024-11-20T22:35:17,032 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:17,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:17,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:17,062 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142117017 2024-11-20T22:35:17,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173)] 2024-11-20T22:35:17,062 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b5fed784dcd7a622aa63b0e259d51d65, NAME => 'testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:17,063 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:17,063 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,063 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,064 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,065 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName a 2024-11-20T22:35:17,065 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,072 DEBUG [StoreFileOpener-b5fed784dcd7a622aa63b0e259d51d65-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:17,072 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ 2024-11-20T22:35:17,077 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7466fe3b57c340d4a27a73f4e11ce69d 2024-11-20T22:35:17,081 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 2024-11-20T22:35:17,086 DEBUG [StoreFileOpener-b5fed784dcd7a622aa63b0e259d51d65-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:17,086 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ 2024-11-20T22:35:17,091 DEBUG [StoreFileOpener-b5fed784dcd7a622aa63b0e259d51d65-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ff66323963734972b096084abbfd1a0c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T22:35:17,092 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ 2024-11-20T22:35:17,092 WARN [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4643c778 2024-11-20T22:35:17,092 WARN [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4643c778 2024-11-20T22:35:17,092 WARN [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4643c778 2024-11-20T22:35:17,092 WARN [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4643c778 2024-11-20T22:35:17,092 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_] to archive 2024-11-20T22:35:17,093 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T22:35:17,095 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_ 2024-11-20T22:35:17,096 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/7ed05e6088ab4f4ca9f57d0b522e4cb0 2024-11-20T22:35:17,097 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_ 2024-11-20T22:35:17,099 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/a/ff66323963734972b096084abbfd1a0c_SeqId_4_ 2024-11-20T22:35:17,099 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,099 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,100 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName b 2024-11-20T22:35:17,100 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,101 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,101 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,103 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5fed784dcd7a622aa63b0e259d51d65 columnFamilyName c 2024-11-20T22:35:17,103 DEBUG [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,104 INFO [StoreOpener-b5fed784dcd7a622aa63b0e259d51d65-1 {}] regionserver.HStore(327): Store=b5fed784dcd7a622aa63b0e259d51d65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,104 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,107 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,107 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 2024-11-20T22:35:17,134 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:17,136 DEBUG [Time-limited test {}] regionserver.HRegion(5836): b5fed784dcd7a622aa63b0e259d51d65 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "b5fed784dcd7a622aa63b0e259d51d65" family_name: "a" compaction_input: "7ed05e6088ab4f4ca9f57d0b522e4cb0" compaction_input: "cb9a449e036d4d9d88fcb55ee4a76cc0_SeqId_4_" compaction_input: "268a2cd2bf684ea2aac0b87735d5532d_SeqId_4_" compaction_input: "ff66323963734972b096084abbfd1a0c_SeqId_4_" compaction_output: "7466fe3b57c340d4a27a73f4e11ce69d" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-20T22:35:17,136 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-20T22:35:17,136 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 2024-11-20T22:35:17,137 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/0000000000000000008 2024-11-20T22:35:17,138 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,138 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,139 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:17,140 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b5fed784dcd7a622aa63b0e259d51d65 2024-11-20T22:35:17,143 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/b5fed784dcd7a622aa63b0e259d51d65/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T22:35:17,144 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b5fed784dcd7a622aa63b0e259d51d65; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60961141, jitterRate=-0.0916082113981247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:17,144 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b5fed784dcd7a622aa63b0e259d51d65: Writing region info on filesystem at 1732142117063Initializing all the Stores at 1732142117064 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117064Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117064Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117064Cleaning up temporary data from old regions at 1732142117138 (+74 ms)Region opened successfully at 1732142117144 (+6 ms) 2024-11-20T22:35:17,147 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b5fed784dcd7a622aa63b0e259d51d65, disabling compactions & flushes 2024-11-20T22:35:17,147 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:17,147 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:17,147 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. after waiting 0 ms 2024-11-20T22:35:17,147 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:17,148 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732142112457.b5fed784dcd7a622aa63b0e259d51d65. 2024-11-20T22:35:17,148 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b5fed784dcd7a622aa63b0e259d51d65: Waiting for close lock at 1732142117147Disabling compacts and flushes for region at 1732142117147Disabling writes for close at 1732142117147Writing region close event to WAL at 1732142117148 (+1 ms)Closed at 1732142117148 2024-11-20T22:35:17,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741929_1108 (size=175) 2024-11-20T22:35:17,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741929_1108 (size=175) 2024-11-20T22:35:17,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741929_1108 (size=175) 2024-11-20T22:35:17,153 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:17,153 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732142117017) 2024-11-20T22:35:17,166 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=445 (was 436) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_321416589_22 at /127.0.0.1:47926 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46165 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_321416589_22 at /127.0.0.1:46740 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_321416589_22 at /127.0.0.1:36136 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 439) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=684 (was 732) 2024-11-20T22:35:17,166 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T22:35:17,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=683 2024-11-20T22:35:17,181 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T22:35:17,202 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:17,204 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T22:35:17,205 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T22:35:17,208 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59077118, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-59077118, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:17,226 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59077118/hregion-59077118.1732142117209, exclude list is [], retry=0 2024-11-20T22:35:17,229 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:17,229 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:17,230 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:17,256 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59077118/hregion-59077118.1732142117209 2024-11-20T22:35:17,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:17,257 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e27b804f6b0bb45d871bc120931d4742, NAME => 'testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-20T22:35:17,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741931_1110 (size=67) 2024-11-20T22:35:17,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741931_1110 (size=67) 2024-11-20T22:35:17,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741931_1110 (size=67) 2024-11-20T22:35:17,268 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:17,269 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,271 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName a 2024-11-20T22:35:17,271 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,271 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,271 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,273 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName b 2024-11-20T22:35:17,273 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,273 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,274 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,275 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName c 2024-11-20T22:35:17,275 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,275 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,276 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,276 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,277 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,278 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,278 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,279 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:17,280 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,284 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T22:35:17,285 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e27b804f6b0bb45d871bc120931d4742; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60562058, jitterRate=-0.097555011510849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:17,285 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e27b804f6b0bb45d871bc120931d4742: Writing region info on filesystem at 1732142117268Initializing all the Stores at 1732142117269 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117269Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117269Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117269Cleaning up temporary data from old regions at 1732142117278 (+9 ms)Region opened successfully at 1732142117285 (+7 ms) 2024-11-20T22:35:17,285 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e27b804f6b0bb45d871bc120931d4742, disabling compactions & flushes 2024-11-20T22:35:17,285 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,285 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,285 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. after waiting 0 ms 2024-11-20T22:35:17,285 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,286 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,286 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e27b804f6b0bb45d871bc120931d4742: Waiting for close lock at 1732142117285Disabling compacts and flushes for region at 1732142117285Disabling writes for close at 1732142117285Writing region close event to WAL at 1732142117286 (+1 ms)Closed at 1732142117286 2024-11-20T22:35:17,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741930_1109 (size=175) 2024-11-20T22:35:17,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741930_1109 (size=175) 2024-11-20T22:35:17,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741930_1109 (size=175) 2024-11-20T22:35:17,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:17,291 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59077118:(num 1732142117209) 2024-11-20T22:35:17,291 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:17,294 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:17,307 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, exclude list is [], retry=0 2024-11-20T22:35:17,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:17,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:17,311 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:17,336 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 2024-11-20T22:35:17,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:17,336 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e27b804f6b0bb45d871bc120931d4742, NAME => 'testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:17,337 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:17,337 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,337 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,338 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,339 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName a 2024-11-20T22:35:17,339 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,339 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,339 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,340 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName b 2024-11-20T22:35:17,340 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,341 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,341 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,341 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName c 2024-11-20T22:35:17,341 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,342 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,344 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:17,346 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,347 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e27b804f6b0bb45d871bc120931d4742; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67564763, jitterRate=0.006793424487113953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:17,347 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e27b804f6b0bb45d871bc120931d4742: Writing region info on filesystem at 1732142117337Initializing all the Stores at 1732142117338 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117338Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117338Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117338Cleaning up temporary data from old regions at 1732142117344 (+6 ms)Region opened successfully at 1732142117347 (+3 ms) 2024-11-20T22:35:17,354 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e27b804f6b0bb45d871bc120931d4742 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T22:35:17,369 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/90385a5c719d43cda0601908456dff93 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732142117347/Put/seqid=0 2024-11-20T22:35:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741933_1112 (size=5958) 2024-11-20T22:35:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741933_1112 (size=5958) 2024-11-20T22:35:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741933_1112 (size=5958) 2024-11-20T22:35:17,379 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/90385a5c719d43cda0601908456dff93 2024-11-20T22:35:17,397 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/90385a5c719d43cda0601908456dff93 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/90385a5c719d43cda0601908456dff93 2024-11-20T22:35:17,403 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/90385a5c719d43cda0601908456dff93, entries=10, sequenceid=13, filesize=5.8 K 2024-11-20T22:35:17,404 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e27b804f6b0bb45d871bc120931d4742 in 49ms, sequenceid=13, compaction requested=false 2024-11-20T22:35:17,404 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e27b804f6b0bb45d871bc120931d4742: 2024-11-20T22:35:17,419 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e27b804f6b0bb45d871bc120931d4742, disabling compactions & flushes 2024-11-20T22:35:17,419 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,419 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,419 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. after waiting 0 ms 2024-11-20T22:35:17,419 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,419 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,420 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:17,420 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e27b804f6b0bb45d871bc120931d4742: Waiting for close lock at 1732142117419Disabling compacts and flushes for region at 1732142117419Disabling writes for close at 1732142117419Writing region close event to WAL at 1732142117419Closed at 1732142117419 2024-11-20T22:35:17,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741932_1111 (size=6243) 2024-11-20T22:35:17,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741932_1111 (size=6243) 2024-11-20T22:35:17,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741932_1111 (size=6243) 2024-11-20T22:35:17,437 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, size=6.1 K (6243bytes) 2024-11-20T22:35:17,437 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 2024-11-20T22:35:17,438 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 after 1ms 2024-11-20T22:35:17,463 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:17,463 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 took 26ms 2024-11-20T22:35:17,466 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 so closing down 2024-11-20T22:35:17,466 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:17,467 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732142117294.temp 2024-11-20T22:35:17,495 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp 2024-11-20T22:35:17,496 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741934_1113 (size=5575) 2024-11-20T22:35:17,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741934_1113 (size=5575) 2024-11-20T22:35:17,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741934_1113 (size=5575) 2024-11-20T22:35:17,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:17,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 2024-11-20T22:35:17,504 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-20T22:35:17,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, size=6.1 K (6243bytes) at 1732142117437Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 so closing down at 1732142117466 (+29 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp at 1732142117495 (+29 ms)3 split writer threads finished at 1732142117496 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732142117502 (+6 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000003-wal.1732142117294.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 at 1732142117504 (+2 ms)Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732142117504 2024-11-20T22:35:17,506 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117294 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142117294 2024-11-20T22:35:17,507 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 2024-11-20T22:35:17,507 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:17,508 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:17,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, exclude list is [], retry=0 2024-11-20T22:35:17,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:17,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:17,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:17,549 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 2024-11-20T22:35:17,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477),(127.0.0.1/127.0.0.1:39567:39567)] 2024-11-20T22:35:17,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e27b804f6b0bb45d871bc120931d4742, NAME => 'testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.', STARTKEY => '', ENDKEY => ''} 2024-11-20T22:35:17,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:17,550 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,550 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,551 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,552 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName a 2024-11-20T22:35:17,552 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,559 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/90385a5c719d43cda0601908456dff93 2024-11-20T22:35:17,560 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,560 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,561 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName b 2024-11-20T22:35:17,561 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,561 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,561 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,562 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName c 2024-11-20T22:35:17,562 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:17,563 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:17,563 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,565 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,565 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 2024-11-20T22:35:17,591 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:17,593 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 2024-11-20T22:35:17,593 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e27b804f6b0bb45d871bc120931d4742 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-20T22:35:17,608 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/7570edc81bd84205b73510e37648c6b5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732142117404/Put/seqid=0 2024-11-20T22:35:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741936_1115 (size=5958) 2024-11-20T22:35:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741936_1115 (size=5958) 2024-11-20T22:35:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741936_1115 (size=5958) 2024-11-20T22:35:17,615 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/7570edc81bd84205b73510e37648c6b5 2024-11-20T22:35:17,634 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/6e447e28d9b74ca5a2a90daf10d26901 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732142117410/Put/seqid=0 2024-11-20T22:35:17,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741937_1116 (size=5958) 2024-11-20T22:35:17,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741937_1116 (size=5958) 2024-11-20T22:35:17,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741937_1116 (size=5958) 2024-11-20T22:35:17,641 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/6e447e28d9b74ca5a2a90daf10d26901 2024-11-20T22:35:17,647 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/7570edc81bd84205b73510e37648c6b5 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/b/7570edc81bd84205b73510e37648c6b5 2024-11-20T22:35:17,652 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/b/7570edc81bd84205b73510e37648c6b5, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T22:35:17,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/6e447e28d9b74ca5a2a90daf10d26901 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/c/6e447e28d9b74ca5a2a90daf10d26901 2024-11-20T22:35:17,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/c/6e447e28d9b74ca5a2a90daf10d26901, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T22:35:17,658 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e27b804f6b0bb45d871bc120931d4742 in 65ms, sequenceid=35, compaction requested=false; wal=null 2024-11-20T22:35:17,659 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000035 2024-11-20T22:35:17,660 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,660 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,661 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:17,662 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:17,665 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-20T22:35:17,666 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e27b804f6b0bb45d871bc120931d4742; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74341891, jitterRate=0.10778050124645233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:17,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e27b804f6b0bb45d871bc120931d4742: Writing region info on filesystem at 1732142117550Initializing all the Stores at 1732142117551 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117551Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117551Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142117551Obtaining lock to block concurrent updates at 1732142117593 (+42 ms)Preparing flush snapshotting stores in e27b804f6b0bb45d871bc120931d4742 at 1732142117593Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732142117593Flushing stores of testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. at 1732142117593Flushing e27b804f6b0bb45d871bc120931d4742/b: creating writer at 1732142117593Flushing e27b804f6b0bb45d871bc120931d4742/b: appending metadata at 1732142117607 (+14 ms)Flushing e27b804f6b0bb45d871bc120931d4742/b: closing flushed file at 1732142117607Flushing e27b804f6b0bb45d871bc120931d4742/c: creating writer at 1732142117620 (+13 ms)Flushing e27b804f6b0bb45d871bc120931d4742/c: appending metadata at 1732142117633 (+13 ms)Flushing e27b804f6b0bb45d871bc120931d4742/c: closing flushed file at 1732142117633Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@478b9ec0: reopening flushed file at 1732142117646 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@575ee5df: reopening flushed file at 1732142117652 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e27b804f6b0bb45d871bc120931d4742 in 65ms, sequenceid=35, compaction requested=false; wal=null at 1732142117658 (+6 ms)Cleaning up temporary data from old regions at 1732142117660 (+2 ms)Region opened successfully at 1732142117666 (+6 ms) 2024-11-20T22:35:17,738 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, size=0 (0bytes) 2024-11-20T22:35:17,738 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 might be still open, length is 0 2024-11-20T22:35:17,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 2024-11-20T22:35:17,738 WARN [IPC Server handler 4 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-20T22:35:17,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 after 0ms 2024-11-20T22:35:20,458 INFO [master/9f31b4695a16:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T22:35:20,459 INFO [master/9f31b4695a16:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T22:35:20,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:46794 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:45551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46794 dst: /127.0.0.1:45551 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45551 remote=/127.0.0.1:46794]. Total timeout mills is 60000, 57177 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:20,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:47970 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36643:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47970 dst: /127.0.0.1:36643 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:20,522 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-428143787_22 at /127.0.0.1:36210 [Receiving block BP-1724098941-172.17.0.2-1732142071568:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37697:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36210 dst: /127.0.0.1:37697 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741935_1117 (size=5567) 2024-11-20T22:35:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741935_1117 (size=5567) 2024-11-20T22:35:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741935_1117 (size=5567) 2024-11-20T22:35:21,740 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 after 4002ms 2024-11-20T22:35:21,767 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:21,767 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 took 4030ms 2024-11-20T22:35:21,769 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509; continuing. 2024-11-20T22:35:21,769 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 so closing down 2024-11-20T22:35:21,770 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T22:35:21,771 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732142117509.temp 2024-11-20T22:35:21,800 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp 2024-11-20T22:35:21,801 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T22:35:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741938_1118 (size=5575) 2024-11-20T22:35:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741938_1118 (size=5575) 2024-11-20T22:35:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741938_1118 (size=5575) 2024-11-20T22:35:21,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T22:35:21,816 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 2024-11-20T22:35:21,816 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T22:35:21,816 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, size=0 (0bytes) at 1732142117738Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 so closing down at 1732142121769 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp at 1732142121800 (+31 ms)3 split writer threads finished at 1732142121801 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732142121815 (+14 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000037-wal.1732142117509.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 at 1732142121816 (+1 ms)Processed 30 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509, size=0, length=0, corrupted=false, cancelled=false at 1732142121816 2024-11-20T22:35:21,818 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 to hdfs://localhost:46165/hbase/oldWALs/wal.1732142117509 2024-11-20T22:35:21,819 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 2024-11-20T22:35:21,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T22:35:21,820 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-20T22:35:21,838 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142121821, exclude list is [], retry=0 2024-11-20T22:35:21,841 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37697,DS-cef45184-81f1-4e78-82eb-3788ee684bc1,DISK] 2024-11-20T22:35:21,841 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45551,DS-0be96184-fe36-4f1d-8e95-83bd78555f17,DISK] 2024-11-20T22:35:21,841 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36643,DS-f243a7d1-9057-4c43-8520-c365588218cf,DISK] 2024-11-20T22:35:21,880 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142121821 2024-11-20T22:35:21,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39567:39567),(127.0.0.1/127.0.0.1:34173:34173),(127.0.0.1/127.0.0.1:36477:36477)] 2024-11-20T22:35:21,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T22:35:21,886 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,887 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName a 2024-11-20T22:35:21,887 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:21,895 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/90385a5c719d43cda0601908456dff93 2024-11-20T22:35:21,895 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:21,895 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,896 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName b 2024-11-20T22:35:21,896 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:21,902 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/b/7570edc81bd84205b73510e37648c6b5 2024-11-20T22:35:21,902 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:21,902 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,903 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e27b804f6b0bb45d871bc120931d4742 columnFamilyName c 2024-11-20T22:35:21,903 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T22:35:21,916 DEBUG [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/c/6e447e28d9b74ca5a2a90daf10d26901 2024-11-20T22:35:21,916 INFO [StoreOpener-e27b804f6b0bb45d871bc120931d4742-1 {}] regionserver.HStore(327): Store=e27b804f6b0bb45d871bc120931d4742/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T22:35:21,916 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,919 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:21,920 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 2024-11-20T22:35:21,950 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T22:35:21,954 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 2024-11-20T22:35:21,954 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e27b804f6b0bb45d871bc120931d4742 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T22:35:21,970 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/c677f742336242c9b4953f496e6961e0 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732142117672/Put/seqid=0 2024-11-20T22:35:21,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741940_1120 (size=5958) 2024-11-20T22:35:21,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741940_1120 (size=5958) 2024-11-20T22:35:21,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741940_1120 (size=5958) 2024-11-20T22:35:21,981 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/c677f742336242c9b4953f496e6961e0 2024-11-20T22:35:22,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/b1c13550f61744d18f5f4358bfbd5e97 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732142117680/Put/seqid=0 2024-11-20T22:35:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741941_1121 (size=5958) 2024-11-20T22:35:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741941_1121 (size=5958) 2024-11-20T22:35:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741941_1121 (size=5958) 2024-11-20T22:35:22,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/b1c13550f61744d18f5f4358bfbd5e97 2024-11-20T22:35:22,050 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/d3ea0ac3da044e929eead8d55ed9293d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732142117687/Put/seqid=0 2024-11-20T22:35:22,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741942_1122 (size=5958) 2024-11-20T22:35:22,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741942_1122 (size=5958) 2024-11-20T22:35:22,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741942_1122 (size=5958) 2024-11-20T22:35:22,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/d3ea0ac3da044e929eead8d55ed9293d 2024-11-20T22:35:22,063 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/a/c677f742336242c9b4953f496e6961e0 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/c677f742336242c9b4953f496e6961e0 2024-11-20T22:35:22,070 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/a/c677f742336242c9b4953f496e6961e0, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T22:35:22,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/b/b1c13550f61744d18f5f4358bfbd5e97 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/b/b1c13550f61744d18f5f4358bfbd5e97 2024-11-20T22:35:22,076 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/b/b1c13550f61744d18f5f4358bfbd5e97, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T22:35:22,077 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/.tmp/c/d3ea0ac3da044e929eead8d55ed9293d as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/c/d3ea0ac3da044e929eead8d55ed9293d 2024-11-20T22:35:22,083 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/c/d3ea0ac3da044e929eead8d55ed9293d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T22:35:22,084 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e27b804f6b0bb45d871bc120931d4742 in 129ms, sequenceid=66, compaction requested=false; wal=null 2024-11-20T22:35:22,086 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/0000000000000000066 2024-11-20T22:35:22,088 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:22,088 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:22,089 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T22:35:22,090 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e27b804f6b0bb45d871bc120931d4742 2024-11-20T22:35:22,093 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/e27b804f6b0bb45d871bc120931d4742/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-20T22:35:22,093 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e27b804f6b0bb45d871bc120931d4742; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63889161, jitterRate=-0.04797731339931488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T22:35:22,094 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e27b804f6b0bb45d871bc120931d4742: Writing region info on filesystem at 1732142121884Initializing all the Stores at 1732142121885 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142121885Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142121886 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732142121886Obtaining lock to block concurrent updates at 1732142121954 (+68 ms)Preparing flush snapshotting stores in e27b804f6b0bb45d871bc120931d4742 at 1732142121954Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732142121955 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. at 1732142121955Flushing e27b804f6b0bb45d871bc120931d4742/a: creating writer at 1732142121955Flushing e27b804f6b0bb45d871bc120931d4742/a: appending metadata at 1732142121969 (+14 ms)Flushing e27b804f6b0bb45d871bc120931d4742/a: closing flushed file at 1732142121969Flushing e27b804f6b0bb45d871bc120931d4742/b: creating writer at 1732142121987 (+18 ms)Flushing e27b804f6b0bb45d871bc120931d4742/b: appending metadata at 1732142122004 (+17 ms)Flushing e27b804f6b0bb45d871bc120931d4742/b: closing flushed file at 1732142122004Flushing e27b804f6b0bb45d871bc120931d4742/c: creating writer at 1732142122027 (+23 ms)Flushing e27b804f6b0bb45d871bc120931d4742/c: appending metadata at 1732142122049 (+22 ms)Flushing e27b804f6b0bb45d871bc120931d4742/c: closing flushed file at 1732142122049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43e58fbb: reopening flushed file at 1732142122062 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13054e3f: reopening flushed file at 1732142122070 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c20370e: reopening flushed file at 1732142122077 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e27b804f6b0bb45d871bc120931d4742 in 129ms, sequenceid=66, compaction requested=false; wal=null at 1732142122084 (+7 ms)Cleaning up temporary data from old regions at 1732142122088 (+4 ms)Region opened successfully at 1732142122094 (+6 ms) 2024-11-20T22:35:22,107 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e27b804f6b0bb45d871bc120931d4742, disabling compactions & flushes 2024-11-20T22:35:22,107 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:22,107 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:22,107 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. after waiting 0 ms 2024-11-20T22:35:22,107 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:22,108 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732142117203.e27b804f6b0bb45d871bc120931d4742. 2024-11-20T22:35:22,109 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e27b804f6b0bb45d871bc120931d4742: Waiting for close lock at 1732142122107Disabling compacts and flushes for region at 1732142122107Disabling writes for close at 1732142122107Writing region close event to WAL at 1732142122108 (+1 ms)Closed at 1732142122108 2024-11-20T22:35:22,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741939_1119 (size=175) 2024-11-20T22:35:22,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741939_1119 (size=175) 2024-11-20T22:35:22,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741939_1119 (size=175) 2024-11-20T22:35:22,113 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T22:35:22,113 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732142121821) 2024-11-20T22:35:22,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446 (was 445) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-87899331_22 at /127.0.0.1:47994 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-87899331_22 at /127.0.0.1:35274 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-87899331_22 at /127.0.0.1:49534 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46165 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=647 (was 683) 2024-11-20T22:35:22,128 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-20T22:35:22,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T22:35:22,128 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T22:35:22,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T22:35:22,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T22:35:22,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T22:35:22,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1122168874, stopped=false 2024-11-20T22:35:22,130 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9f31b4695a16,40769,1732142076492 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:35:22,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T22:35:22,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:35:22,202 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T22:35:22,202 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T22:35:22,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:35:22,202 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9f31b4695a16,43815,1732142077481' ***** 2024-11-20T22:35:22,202 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T22:35:22,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:35:22,203 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9f31b4695a16,36989,1732142077539' ***** 2024-11-20T22:35:22,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T22:35:22,203 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(959): stopping server 9f31b4695a16,43815,1732142077481 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9f31b4695a16:43815. 2024-11-20T22:35:22,203 DEBUG [RS:1;9f31b4695a16:43815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T22:35:22,203 DEBUG [RS:1;9f31b4695a16:43815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,203 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(976): stopping server 9f31b4695a16,43815,1732142077481; all regions closed. 2024-11-20T22:35:22,203 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T22:35:22,204 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:22,204 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(959): stopping server 9f31b4695a16,36989,1732142077539 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9f31b4695a16:36989. 2024-11-20T22:35:22,204 DEBUG [RS:2;9f31b4695a16:36989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T22:35:22,204 DEBUG [RS:2;9f31b4695a16:36989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,204 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e972e424facefaa32be2f83dd0a8f47, disabling compactions & flushes 2024-11-20T22:35:22,204 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:22,204 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T22:35:22,205 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T22:35:22,205 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:22,205 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T22:35:22,205 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. after waiting 0 ms 2024-11-20T22:35:22,205 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:22,205 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T22:35:22,205 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T22:35:22,205 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1325): Online Regions={5e972e424facefaa32be2f83dd0a8f47=testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T22:35:22,205 DEBUG [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5e972e424facefaa32be2f83dd0a8f47 2024-11-20T22:35:22,206 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T22:35:22,206 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T22:35:22,206 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T22:35:22,206 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T22:35:22,206 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T22:35:22,207 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-20T22:35:22,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741833_1009 (size=2764) 2024-11-20T22:35:22,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741833_1009 (size=2764) 2024-11-20T22:35:22,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741833_1009 (size=2764) 2024-11-20T22:35:22,214 DEBUG [RS:1;9f31b4695a16:43815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs 2024-11-20T22:35:22,214 INFO [RS:1;9f31b4695a16:43815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f31b4695a16%2C43815%2C1732142077481:(num 1732142079500) 2024-11-20T22:35:22,215 DEBUG [RS:1;9f31b4695a16:43815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,215 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:35:22,215 INFO [RS:1;9f31b4695a16:43815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] hbase.ChoreService(370): Chore service for: regionserver/9f31b4695a16:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T22:35:22,216 INFO [RS:1;9f31b4695a16:43815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43815 2024-11-20T22:35:22,217 INFO [regionserver/9f31b4695a16:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T22:35:22,219 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5e972e424facefaa32be2f83dd0a8f47/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-20T22:35:22,220 INFO [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:22,220 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e972e424facefaa32be2f83dd0a8f47: Waiting for close lock at 1732142122204Running coprocessor pre-close hooks at 1732142122204Disabling compacts and flushes for region at 1732142122204Disabling writes for close at 1732142122205 (+1 ms)Writing region close event to WAL at 1732142122207 (+2 ms)Running coprocessor post-close hooks at 1732142122219 (+12 ms)Closed at 1732142122219 2024-11-20T22:35:22,220 DEBUG [RS_CLOSE_REGION-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47. 2024-11-20T22:35:22,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T22:35:22,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9f31b4695a16,43815,1732142077481 2024-11-20T22:35:22,228 INFO [RS:1;9f31b4695a16:43815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T22:35:22,228 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9f31b4695a16,43815,1732142077481] 2024-11-20T22:35:22,232 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/95dd38345ecb409bba54464c0ca8a1be is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732142095753.5e972e424facefaa32be2f83dd0a8f47./info:regioninfo/1732142100128/Put/seqid=0 2024-11-20T22:35:22,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741943_1123 (size=6778) 2024-11-20T22:35:22,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741943_1123 (size=6778) 2024-11-20T22:35:22,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741943_1123 (size=6778) 2024-11-20T22:35:22,244 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/95dd38345ecb409bba54464c0ca8a1be 2024-11-20T22:35:22,249 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9f31b4695a16,43815,1732142077481 already deleted, retry=false 2024-11-20T22:35:22,250 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9f31b4695a16,43815,1732142077481 expired; onlineServers=1 2024-11-20T22:35:22,255 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/.tmp/info/95dd38345ecb409bba54464c0ca8a1be as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/info/95dd38345ecb409bba54464c0ca8a1be 2024-11-20T22:35:22,257 INFO [regionserver/9f31b4695a16:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:35:22,261 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/info/95dd38345ecb409bba54464c0ca8a1be, entries=8, sequenceid=23, filesize=6.6 K 2024-11-20T22:35:22,262 INFO [regionserver/9f31b4695a16:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:35:22,262 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=23, compaction requested=false 2024-11-20T22:35:22,267 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-20T22:35:22,267 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T22:35:22,267 INFO [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T22:35:22,267 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732142122206Running coprocessor pre-close hooks at 1732142122206Disabling compacts and flushes for region at 1732142122206Disabling writes for close at 1732142122206Obtaining lock to block concurrent updates at 1732142122207 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732142122207Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732142122207Flushing stores of hbase:meta,,1.1588230740 at 1732142122220 (+13 ms)Flushing 1588230740/info: creating writer at 1732142122220Flushing 1588230740/info: appending metadata at 1732142122231 (+11 ms)Flushing 1588230740/info: closing flushed file at 1732142122231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e7b05f: reopening flushed file at 1732142122254 (+23 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=23, compaction requested=false at 1732142122262 (+8 ms)Writing region close event to WAL at 1732142122263 (+1 ms)Running coprocessor post-close hooks at 1732142122267 (+4 ms)Closed at 1732142122267 2024-11-20T22:35:22,268 DEBUG [RS_CLOSE_META-regionserver/9f31b4695a16:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T22:35:22,338 INFO [RS:1;9f31b4695a16:43815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T22:35:22,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,338 INFO [RS:1;9f31b4695a16:43815 {}] regionserver.HRegionServer(1031): Exiting; stopping=9f31b4695a16,43815,1732142077481; zookeeper connection closed. 2024-11-20T22:35:22,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43815-0x1015a628ac10002, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,339 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23f4024f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23f4024f 2024-11-20T22:35:22,388 INFO [regionserver/9f31b4695a16:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T22:35:22,389 INFO [regionserver/9f31b4695a16:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T22:35:22,406 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(976): stopping server 9f31b4695a16,36989,1732142077539; all regions closed. 2024-11-20T22:35:22,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741893_1071 (size=2625) 2024-11-20T22:35:22,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741893_1071 (size=2625) 2024-11-20T22:35:22,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741893_1071 (size=2625) 2024-11-20T22:35:22,414 DEBUG [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs 2024-11-20T22:35:22,414 INFO [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f31b4695a16%2C36989%2C1732142077539.meta:.meta(num 1732142099104) 2024-11-20T22:35:22,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741834_1010 (size=1032) 2024-11-20T22:35:22,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741834_1010 (size=1032) 2024-11-20T22:35:22,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741834_1010 (size=1032) 2024-11-20T22:35:22,421 DEBUG [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/oldWALs 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f31b4695a16%2C36989%2C1732142077539:(num 1732142079497) 2024-11-20T22:35:22,421 DEBUG [RS:2;9f31b4695a16:36989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] hbase.ChoreService(370): Chore service for: regionserver/9f31b4695a16:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T22:35:22,421 INFO [regionserver/9f31b4695a16:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T22:35:22,421 INFO [RS:2;9f31b4695a16:36989 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36989 2024-11-20T22:35:22,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T22:35:22,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9f31b4695a16,36989,1732142077539 2024-11-20T22:35:22,438 INFO [RS:2;9f31b4695a16:36989 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T22:35:22,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9f31b4695a16,36989,1732142077539] 2024-11-20T22:35:22,459 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9f31b4695a16,36989,1732142077539 already deleted, retry=false 2024-11-20T22:35:22,459 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9f31b4695a16,36989,1732142077539 expired; onlineServers=0 2024-11-20T22:35:22,459 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9f31b4695a16,40769,1732142076492' ***** 2024-11-20T22:35:22,459 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T22:35:22,459 INFO [M:0;9f31b4695a16:40769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T22:35:22,459 INFO [M:0;9f31b4695a16:40769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T22:35:22,459 DEBUG [M:0;9f31b4695a16:40769 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T22:35:22,459 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T22:35:22,459 DEBUG [M:0;9f31b4695a16:40769 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T22:35:22,459 DEBUG [master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.large.0-1732142079067 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.large.0-1732142079067,5,FailOnTimeoutGroup] 2024-11-20T22:35:22,460 DEBUG [master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.small.0-1732142079069 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f31b4695a16:0:becomeActiveMaster-HFileCleaner.small.0-1732142079069,5,FailOnTimeoutGroup] 2024-11-20T22:35:22,460 INFO [M:0;9f31b4695a16:40769 {}] hbase.ChoreService(370): Chore service for: master/9f31b4695a16:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T22:35:22,460 INFO [M:0;9f31b4695a16:40769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T22:35:22,460 DEBUG [M:0;9f31b4695a16:40769 {}] master.HMaster(1795): Stopping service threads 2024-11-20T22:35:22,460 INFO [M:0;9f31b4695a16:40769 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T22:35:22,460 INFO [M:0;9f31b4695a16:40769 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T22:35:22,461 INFO [M:0;9f31b4695a16:40769 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T22:35:22,461 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T22:35:22,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T22:35:22,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T22:35:22,469 DEBUG [M:0;9f31b4695a16:40769 {}] zookeeper.ZKUtil(347): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T22:35:22,469 WARN [M:0;9f31b4695a16:40769 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T22:35:22,470 INFO [M:0;9f31b4695a16:40769 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/.lastflushedseqids 2024-11-20T22:35:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741944_1124 (size=119) 2024-11-20T22:35:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741944_1124 (size=119) 2024-11-20T22:35:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741944_1124 (size=119) 2024-11-20T22:35:22,483 INFO [M:0;9f31b4695a16:40769 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T22:35:22,483 INFO [M:0;9f31b4695a16:40769 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T22:35:22,483 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T22:35:22,483 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:35:22,483 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:35:22,483 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T22:35:22,483 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:35:22,483 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-20T22:35:22,499 DEBUG [M:0;9f31b4695a16:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8118e7c4e4204e36b361a00d13dd0061 is 82, key is hbase:meta,,1/info:regioninfo/1732142099348/Put/seqid=0 2024-11-20T22:35:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741945_1125 (size=6063) 2024-11-20T22:35:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741945_1125 (size=6063) 2024-11-20T22:35:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741945_1125 (size=6063) 2024-11-20T22:35:22,504 INFO [M:0;9f31b4695a16:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8118e7c4e4204e36b361a00d13dd0061 2024-11-20T22:35:22,523 DEBUG [M:0;9f31b4695a16:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1131523f378b437a8f4682843fab207f is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732142096199/Put/seqid=0 2024-11-20T22:35:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741946_1126 (size=7908) 2024-11-20T22:35:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741946_1126 (size=7908) 2024-11-20T22:35:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741946_1126 (size=7908) 2024-11-20T22:35:22,529 INFO [M:0;9f31b4695a16:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1131523f378b437a8f4682843fab207f 2024-11-20T22:35:22,533 INFO [M:0;9f31b4695a16:40769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1131523f378b437a8f4682843fab207f 2024-11-20T22:35:22,546 DEBUG [M:0;9f31b4695a16:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bd34e506a0d427ebd04ac12dc5b03e0 is 69, key is 9f31b4695a16,36989,1732142077539/rs:state/1732142079178/Put/seqid=0 2024-11-20T22:35:22,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,548 INFO [RS:2;9f31b4695a16:36989 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T22:35:22,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36989-0x1015a628ac10003, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,548 INFO [RS:2;9f31b4695a16:36989 {}] regionserver.HRegionServer(1031): Exiting; stopping=9f31b4695a16,36989,1732142077539; zookeeper connection closed. 2024-11-20T22:35:22,549 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50ae14ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50ae14ae 2024-11-20T22:35:22,549 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T22:35:22,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741947_1127 (size=5445) 2024-11-20T22:35:22,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741947_1127 (size=5445) 2024-11-20T22:35:22,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741947_1127 (size=5445) 2024-11-20T22:35:22,555 INFO [M:0;9f31b4695a16:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bd34e506a0d427ebd04ac12dc5b03e0 2024-11-20T22:35:22,560 INFO [M:0;9f31b4695a16:40769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bd34e506a0d427ebd04ac12dc5b03e0 2024-11-20T22:35:22,561 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8118e7c4e4204e36b361a00d13dd0061 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8118e7c4e4204e36b361a00d13dd0061 2024-11-20T22:35:22,567 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8118e7c4e4204e36b361a00d13dd0061, entries=14, sequenceid=207, filesize=5.9 K 2024-11-20T22:35:22,568 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1131523f378b437a8f4682843fab207f as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1131523f378b437a8f4682843fab207f 2024-11-20T22:35:22,573 INFO [M:0;9f31b4695a16:40769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1131523f378b437a8f4682843fab207f 2024-11-20T22:35:22,573 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1131523f378b437a8f4682843fab207f, entries=21, sequenceid=207, filesize=7.7 K 2024-11-20T22:35:22,574 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bd34e506a0d427ebd04ac12dc5b03e0 as hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bd34e506a0d427ebd04ac12dc5b03e0 2024-11-20T22:35:22,579 INFO [M:0;9f31b4695a16:40769 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bd34e506a0d427ebd04ac12dc5b03e0 2024-11-20T22:35:22,579 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/100aa28a-357f-96b2-6bcc-a4cb97225647/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bd34e506a0d427ebd04ac12dc5b03e0, entries=3, sequenceid=207, filesize=5.3 K 2024-11-20T22:35:22,580 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false 2024-11-20T22:35:22,582 INFO [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T22:35:22,582 DEBUG [M:0;9f31b4695a16:40769 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732142122483Disabling compacts and flushes for region at 1732142122483Disabling writes for close at 1732142122483Obtaining lock to block concurrent updates at 1732142122483Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732142122483Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1732142122484 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732142122484Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732142122484Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732142122498 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732142122498Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732142122508 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732142122522 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732142122522Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732142122533 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732142122546 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732142122546Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@103b4544: reopening flushed file at 1732142122560 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43eed5d5: reopening flushed file at 1732142122567 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bb9ebda: reopening flushed file at 1732142122573 (+6 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false at 1732142122580 (+7 ms)Writing region close event to WAL at 1732142122582 (+2 ms)Closed at 1732142122582 2024-11-20T22:35:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36643 is added to blk_1073741830_1006 (size=102174) 2024-11-20T22:35:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45551 is added to blk_1073741830_1006 (size=102174) 2024-11-20T22:35:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37697 is added to blk_1073741830_1006 (size=102174) 2024-11-20T22:35:22,586 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T22:35:22,586 INFO [M:0;9f31b4695a16:40769 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T22:35:22,586 INFO [M:0;9f31b4695a16:40769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40769 2024-11-20T22:35:22,587 INFO [M:0;9f31b4695a16:40769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T22:35:22,696 INFO [M:0;9f31b4695a16:40769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T22:35:22,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x1015a628ac10000, quorum=127.0.0.1:62486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T22:35:22,702 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142111191 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142111191 (inode 16688) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732142101797/wal.1732142111191 (inode 16688) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T22:35:22,705 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142101553 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142101553 (inode 16665) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732142100719/wal.1732142101553 (inode 16665) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T22:35:22,705 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732142090490/wal.1732142091154 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:22,707 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142112196 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142112196 (inode 16714) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732142111458/wal.1732142112196 (inode 16714) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T22:35:22,708 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732142112456/wal.1732142112646 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:22,708 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732142117201/wal.1732142117509 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:22,709 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732142082284/wal.1732142082409 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T22:35:22,711 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal.1732142082047 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal.1732142082047 (inode 16485) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732142081587/wal.1732142082047 (inode 16485) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T22:35:22,714 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal.1732142081215 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal.1732142081215 (inode 16462) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732142080633/wal.1732142081215 (inode 16462) Holder DFSClient_NONMAPREDUCE_-428143787_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T22:35:22,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:35:22,721 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T22:35:22,721 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T22:35:22,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T22:35:22,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,STOPPED} 2024-11-20T22:35:22,724 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T22:35:22,724 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T22:35:22,724 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1724098941-172.17.0.2-1732142071568 (Datanode Uuid d6591d0a-a4f6-4a72-8c20-0bf3d2fc991d) service to localhost/127.0.0.1:46165 2024-11-20T22:35:22,724 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T22:35:22,725 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data5/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,725 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data6/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,726 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T22:35:22,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:35:22,728 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T22:35:22,729 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T22:35:22,729 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T22:35:22,729 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,STOPPED} 2024-11-20T22:35:22,730 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T22:35:22,730 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T22:35:22,730 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1724098941-172.17.0.2-1732142071568 (Datanode Uuid 201b5eee-3952-42bb-873b-5a45722d3c69) service to localhost/127.0.0.1:46165 2024-11-20T22:35:22,730 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T22:35:22,731 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data3/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,731 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data4/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,731 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T22:35:22,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T22:35:22,734 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T22:35:22,734 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T22:35:22,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T22:35:22,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,STOPPED} 2024-11-20T22:35:22,735 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T22:35:22,735 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T22:35:22,735 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T22:35:22,735 WARN [BP-1724098941-172.17.0.2-1732142071568 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1724098941-172.17.0.2-1732142071568 (Datanode Uuid e28c0d50-94dc-4ddc-8035-8a0aa1b81890) service to localhost/127.0.0.1:46165 2024-11-20T22:35:22,736 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data1/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,736 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/cluster_dfdbe74f-50fc-6cbb-4cda-57a5d1b4e947/data/data2/current/BP-1724098941-172.17.0.2-1732142071568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T22:35:22,737 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T22:35:22,744 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T22:35:22,745 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T22:35:22,745 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T22:35:22,745 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T22:35:22,745 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51caf729-32a2-ce03-f426-8e8af6585a80/hadoop.log.dir/,STOPPED} 2024-11-20T22:35:22,754 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T22:35:22,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down